From 0d242f8f5e71cd0027ad5606b9acb73d1c1348c3 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Wed, 17 May 2023 13:43:36 +0200 Subject: [PATCH 01/16] added zarr_io.py example --- legate/examples/zarr_io.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 legate/examples/zarr_io.py diff --git a/legate/examples/zarr_io.py b/legate/examples/zarr_io.py new file mode 100644 index 0000000000..f79cbe6826 --- /dev/null +++ b/legate/examples/zarr_io.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + +import cunumeric as num + +import legate.core +import legate_kvikio.zarr + + +def zarr_io(dirname): + a = num.arange(10000).reshape(100, 100) + + # Write array to a Zarr file by chunks of 10x10. + legate_kvikio.zarr.write_array(a, dirname, chunks=(10, 10)) + + # Block until done writing. + legate.core.get_legate_runtime().issue_execution_fence(block=True) + + # Read array from a Zarr file. + b = legate_kvikio.zarr.read_array(dirname) + + # They should be equal + assert (a == b).all() + + +if __name__ == "__main__": + zarr_io("/tmp/legate-kvikio-zarr-io") From f72f2b4c27fc23f75a8b6838512553339460aa20 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Mon, 22 May 2023 17:12:32 +0200 Subject: [PATCH 02/16] importorskip zarr --- legate/tests/test_basic_io.py | 4 +++- legate/tests/test_zarr.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/legate/tests/test_basic_io.py b/legate/tests/test_basic_io.py index 9c62f89e72..7b7d122861 100644 --- a/legate/tests/test_basic_io.py +++ b/legate/tests/test_basic_io.py @@ -8,7 +8,9 @@ from legate.core import get_legate_runtime from legate_kvikio import CuFile -from legate_kvikio.tile import read_tiles, write_tiles + +read_tiles = pytest.importorskip("legate_kvikio.tile").read_tiles +write_tiles = pytest.importorskip("legate_kvikio.tile").write_tiles num = pytest.importorskip("cunumeric") diff --git a/legate/tests/test_zarr.py b/legate/tests/test_zarr.py index 90016ddb49..d641d4b49e 100644 --- a/legate/tests/test_zarr.py +++ b/legate/tests/test_zarr.py @@ -9,8 +9,9 @@ from numpy.testing import assert_array_equal from legate.core import get_legate_runtime -from legate_kvikio.zarr import read_array, write_array +read_array = pytest.importorskip("legate_kvikio.zarr").read_array +write_array = pytest.importorskip("legate_kvikio.zarr").write_array num = pytest.importorskip("cunumeric") zarr = pytest.importorskip("zarr") From 8f5ef2c4d21ad9264e7acd7603c714cb1e6e5be1 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Mon, 22 May 2023 17:18:27 +0200 Subject: [PATCH 03/16] Impl. read_tiles_by_offsets --- legate/cpp/task_opcodes.hpp | 1 + legate/cpp/tile_io.cpp | 100 +++++++++++++++++++- legate/legate_kvikio/kerchunk.py | 80 ++++++++++++++++ legate/legate_kvikio/library_description.py | 1 + legate/legate_kvikio/tile.py | 38 +++++++- legate/legate_kvikio/zarr.py | 6 +- legate/tests/test_kerchunk.py | 39 ++++++++ 7 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 legate/legate_kvikio/kerchunk.py create mode 100644 legate/tests/test_kerchunk.py diff --git a/legate/cpp/task_opcodes.hpp b/legate/cpp/task_opcodes.hpp index 82575c987c..9920a8c7ac 100644 --- a/legate/cpp/task_opcodes.hpp +++ b/legate/cpp/task_opcodes.hpp @@ -20,5 +20,6 @@ enum TaskOpCode { OP_READ, OP_TILE_WRITE, OP_TILE_READ, + OP_TILE_READ_BY_OFFSETS, OP_NUM_TASK_IDS, // must be last }; diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index ddc9dc7ceb..3986f613c2 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "legate_mapping.hpp" @@ -70,7 +71,7 @@ std::filesystem::path get_file_path(const std::string& dirpath, * * @tparam IsReadOperation Whether the operation is a read or a write operation * @param context The Legate task context - * @param store The Legate store ti read or write + * @param store The Legate store to read or write */ template struct tile_read_write_fn { @@ -105,6 +106,71 @@ struct tile_read_write_fn { } }; +/** + * @brief Flatten the domain point to a 1D point + * + * @param lo_dp Lower point + * @param hi_dp High point + * @param point_dp The domain point to flatten + * @return The flatten domain point + */ +template +size_t linearize(const legate::DomainPoint& lo_dp, + const legate::DomainPoint& hi_dp, + const legate::DomainPoint& point_dp) +{ + legate::Point lo = lo_dp; + legate::Point hi = hi_dp; + legate::Point point = point_dp; + legate::Point extents = hi - lo + legate::Point::ONES(); + size_t idx = 0; + for (int32_t dim = 0; dim < DIM; ++dim) { + idx = idx * extents[dim] + point[dim] - lo[dim]; + } + return idx; +} + +/** + * @brief Functor for tiling read Legate store by offsets from disk using KvikIO + * + * @param context The Legate task context + * @param store The Legate output store + */ +struct tile_read_by_offsets_fn { + template + void operator()(legate::TaskContext& context, legate::Store& store) + { + using DTYPE = legate::legate_type_of; + const auto task_index = context.get_task_index(); + const auto launch_domain = context.get_launch_domain(); + const std::string path = context.scalars().at(0).value(); + legate::Span offsets = context.scalars().at(1).values(); + legate::Span sizes = context.scalars().at(2).values(); + legate::Span tile_shape = context.scalars().at(3).values(); + + // Flatten task index + uint32_t flatten_task_index = 0; + if (!context.is_single_task()) { + flatten_task_index = linearize(launch_domain.lo(), launch_domain.hi(), task_index); + } + + auto shape = store.shape(); + auto shape_volume = shape.volume(); + if (shape_volume == 0) { return; } + size_t nbytes = shape_volume * sizeof(DTYPE); + std::array strides{}; + if (nbytes != sizes[flatten_task_index]) { + throw std::runtime_error("sizes doesn't match the size of the output store"); + } + + // We know that the accessor is contiguous because we set `policy.exact = true` + // in `Mapper::store_mappings()`. + kvikio::FileHandle f(path, "r"); + auto* data = store.write_accessor().ptr(shape, strides.data()); + f.pread(data, nbytes, offsets[flatten_task_index]).get(); + } +}; + } // namespace namespace legate_kvikio { @@ -167,6 +233,37 @@ class TileReadTask : public Task { } }; +/** + * @brief Read a tiled Legate store by offset to disk using KvikIO + * Task signature: + * - scalars: + * - path: std::string + * - offsets: tuple of int64_t + * - sizes: tuple of int64_t + * - tile_shape: tuple of int64_t + * - outputs: + * - buffer: store (any dtype) + * + * NB: the store must be contigues. To make Legate in force this, + * set `policy.exact = true` in `Mapper::store_mappings()`. + * + */ +class TileReadByOffsetsTask + : public Task { + public: + static void cpu_variant(legate::TaskContext& context) + { + legate::Store& store = context.outputs().at(0); + legate::double_dispatch(store.dim(), store.code(), tile_read_by_offsets_fn{}, context, store); + } + + static void gpu_variant(legate::TaskContext& context) + { + // Since KvikIO supports both GPU and CPU memory seamlessly, we reuse the CPU variant. + cpu_variant(context); + } +}; + } // namespace legate_kvikio namespace { @@ -175,6 +272,7 @@ void __attribute__((constructor)) register_tasks() { legate_kvikio::TileWriteTask::register_variants(); legate_kvikio::TileReadTask::register_variants(); + legate_kvikio::TileReadByOffsetsTask::register_variants(); } } // namespace diff --git a/legate/legate_kvikio/kerchunk.py b/legate/legate_kvikio/kerchunk.py new file mode 100644 index 0000000000..8e51c86d28 --- /dev/null +++ b/legate/legate_kvikio/kerchunk.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + +from __future__ import annotations + +import itertools +import math +import pathlib + +import cunumeric +import fsspec +import zarr.core +import zarr.hierarchy +from kerchunk.hdf import SingleHdf5ToZarr + +from .tile import read_tiles_by_offsets +from .zarr import get_padded_array + + +def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndarray: + """Read an HDF5 array from disk using KvikIO + + Notes + ----- + The returned array is padded to make its shape divisible by the shape of + the Zarr chunks on disk (if not already). This means that the returned + Legate store can be larger than the returned cuNumeric array. + + Parameters + ---------- + filepath + File path to the hdf5 file. + + Return + ------ + The cuNumeric array read from disk. + """ + filepath = pathlib.Path(filepath) + + # TODO: look for already generated kerchunk annotations + annotations = SingleHdf5ToZarr(filepath, inline_threshold=0).translate() + + # Load annotations + zarr_group = zarr.open(fsspec.get_mapper("reference://", fo=annotations)) + zarr_ary: zarr.Array = zarr_group[dataset_name] + if zarr_ary.compressor is not None: + raise NotImplementedError("compressor isn't supported") + + # Extract offset and bytes for each chunk + refs = annotations["refs"] + offsets = [] + sizes = [] + for chunk_coord in itertools.product( + *(range(math.ceil(s / c)) for s, c in zip(zarr_ary.shape, zarr_ary.chunks)) + ): + key = zarr_ary._chunk_key(chunk_coord) + _, offset, nbytes = refs[key] + offsets.append(offset) + sizes.append(nbytes) + + padded_ary = get_padded_array(zarr_ary) + if padded_ary is None: + ret = cunumeric.empty(shape=zarr_ary.shape, dtype=zarr_ary.dtype) + read_tiles_by_offsets( + ret, + filepaths=[filepath], + offsets=tuple(offsets), + sizes=tuple(sizes), + tile_shape=zarr_ary.chunks, + ) + else: + read_tiles_by_offsets( + padded_ary, + filepaths=[filepath], + offsets=tuple(offsets), + sizes=tuple(sizes), + tile_shape=zarr_ary.chunks, + ) + ret = padded_ary[tuple(slice(s) for s in zarr_ary.shape)] + return ret diff --git a/legate/legate_kvikio/library_description.py b/legate/legate_kvikio/library_description.py index 67802e4e1d..661b4cc096 100644 --- a/legate/legate_kvikio/library_description.py +++ b/legate/legate_kvikio/library_description.py @@ -45,3 +45,4 @@ class TaskOpCode(IntEnum): READ = description.cffi.OP_READ TILE_WRITE = description.cffi.OP_TILE_WRITE TILE_READ = description.cffi.OP_TILE_READ + TILE_READ_BY_OFFSETS = description.cffi.OP_TILE_READ_BY_OFFSETS diff --git a/legate/legate_kvikio/tile.py b/legate/legate_kvikio/tile.py index dceecdc502..6d63af9043 100644 --- a/legate/legate_kvikio/tile.py +++ b/legate/legate_kvikio/tile.py @@ -4,7 +4,7 @@ from __future__ import annotations import pathlib -from typing import Optional, Tuple +from typing import Iterable, Optional, Tuple import cunumeric @@ -102,3 +102,39 @@ def read_tiles( The start coordinate of the tiles """ return _tile_read_write(TaskOpCode.TILE_READ, ary, dirpath, tile_shape, tile_start) + + +def read_tiles_by_offsets( + ary: cunumeric.ndarray, + filepaths: Iterable[pathlib.Path | str], + offsets: Tuple[int], + sizes: Tuple[int], + tile_shape: Tuple[int], +) -> None: + # TODO; support a filepath per offset/size + assert len(filepaths) == 1, "For now, only a single filepath is supported" + filepath = str(filepaths[0]) + + if any(d % c != 0 for d, c in zip(ary.shape, tile_shape)): + raise ValueError( + f"The tile shape {tile_shape} must be " + f"divisible with the array shape {ary.shape}" + ) + + # Partition the array into even tiles + store_partition = get_legate_store(ary).partition_by_tiling(tile_shape) + + # Use the partition's color shape as the launch shape so there will be + # one task for each tile + launch_shape = store_partition.partition.color_shape + task = context.create_manual_task( + TaskOpCode.TILE_READ_BY_OFFSETS, + launch_domain=Rect(launch_shape), + ) + + task.add_output(store_partition) + task.add_scalar_arg(filepath, types.string) + task.add_scalar_arg(offsets, (types.uint64,)) + task.add_scalar_arg(sizes, (types.uint64,)) + task.add_scalar_arg(tile_shape, (types.uint64,)) + task.execute() diff --git a/legate/legate_kvikio/zarr.py b/legate/legate_kvikio/zarr.py index b0ddbc1be3..f4e0052b23 100644 --- a/legate/legate_kvikio/zarr.py +++ b/legate/legate_kvikio/zarr.py @@ -13,7 +13,7 @@ from .tile import read_tiles, write_tiles -def _get_padded_array(zarr_ary: zarr.Array) -> Optional[cunumeric.ndarray]: +def get_padded_array(zarr_ary: zarr.Array) -> Optional[cunumeric.ndarray]: """Get a padded array that has an shape divisible by `zarr_ary.chunks`. Parameters @@ -74,7 +74,7 @@ def write_array( chunks=chunks, compressor=compressor, ) - padded_ary = _get_padded_array(zarr_ary) + padded_ary = get_padded_array(zarr_ary) if padded_ary is None: write_tiles(ary, dirpath=dirpath, tile_shape=zarr_ary.chunks) else: @@ -108,7 +108,7 @@ def read_array(dirpath: pathlib.Path | str) -> cunumeric.ndarray: if zarr_ary.compressor is not None: raise NotImplementedError("compressor isn't supported") - padded_ary = _get_padded_array(zarr_ary) + padded_ary = get_padded_array(zarr_ary) if padded_ary is None: ret = cunumeric.empty(shape=zarr_ary.shape, dtype=zarr_ary.dtype) read_tiles(ret, dirpath=dirpath, tile_shape=zarr_ary.chunks) diff --git a/legate/tests/test_kerchunk.py b/legate/tests/test_kerchunk.py new file mode 100644 index 0000000000..f159ea8f82 --- /dev/null +++ b/legate/tests/test_kerchunk.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + + +import math + +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +hdf5_read = pytest.importorskip("legate_kvikio.kerchunk").hdf5_read + +num = pytest.importorskip("cunumeric") + +shape_chunks = ( + "shape,chunks", + [ + ((2,), (2,)), + ((5,), (2,)), + ((4, 2), (2, 2)), + ((2, 4), (2, 2)), + ((2, 3), (2, 2)), + ((5, 4, 3, 2), (2, 2, 2, 2)), + ], +) + + +@pytest.mark.parametrize(*shape_chunks) +@pytest.mark.parametrize("dtype", ["u1", "u8", "f8"]) +def test_hdf5_read_array(tmp_path, shape, chunks, dtype): + h5py = pytest.importorskip("h5py") + + filename = tmp_path / "test-file.hdf5" + a = np.arange(math.prod(shape), dtype=dtype).reshape(shape) + with h5py.File(filename, "w") as f: + f.create_dataset("mydataset", chunks=chunks, data=a) + + b = hdf5_read(filename, dataset_name="mydataset") + assert_array_equal(a, b) From 8d4f773a8bc492a5030b2a63203647c4e01efa19 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 23 May 2023 09:49:28 +0200 Subject: [PATCH 04/16] impl. hdf5 example --- legate/examples/hdf5_io.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 legate/examples/hdf5_io.py diff --git a/legate/examples/hdf5_io.py b/legate/examples/hdf5_io.py new file mode 100644 index 0000000000..d8f0a876d9 --- /dev/null +++ b/legate/examples/hdf5_io.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + + +import h5py +import numpy as np + +import legate_kvikio.kerchunk +import legate_kvikio.zarr + + +def hdf5_io(filename): + a = np.arange(10000).reshape((100, 100)) + + # Write array using h5py + with h5py.File(filename, "w") as f: + f.create_dataset("mydataset", chunks=(10, 10), data=a) + + # Read hdf5 file using legate+kerchunk + b = legate_kvikio.kerchunk.hdf5_read(filename, dataset_name="mydataset") + + # They should be equal + assert (a == b).all() + + +if __name__ == "__main__": + hdf5_io("/tmp/legate-kvikio-io.hdf5") From 43c916d2e6d979aff1a9c045e4c27ce81b01ba9d Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 23 May 2023 13:53:57 +0200 Subject: [PATCH 05/16] benchmark --- legate/benchmarks/hdf5_read.py | 185 +++++++++++++++++++++++++++++++++ legate/examples/hdf5_io.py | 1 - 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 legate/benchmarks/hdf5_read.py diff --git a/legate/benchmarks/hdf5_read.py b/legate/benchmarks/hdf5_read.py new file mode 100644 index 0000000000..9e096e8ff9 --- /dev/null +++ b/legate/benchmarks/hdf5_read.py @@ -0,0 +1,185 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + +import argparse +import contextlib +import pathlib +import tempfile +from time import perf_counter as clock +from typing import ContextManager + +import h5py +import numpy as np + +DATASET = "dataset-hdf5-read" + + +def try_open_hdf5_array(filepath, shape, chunks, dtype): + try: + with h5py.File(filepath, "r") as f: + a = f[DATASET] + chunks = chunks or a.chunks + if a.shape == shape and a.chunks == chunks and a.dtype == dtype: + return a + except FileNotFoundError: + pass + return None + + +def create_hdf5_array(filepath, shape, chunks, dtype=np.float64) -> None: + ret = try_open_hdf5_array(filepath, shape, chunks, dtype) + if ret is None: + # Write array using h5py + with h5py.File(filepath, "w") as f: + f.create_dataset(DATASET, chunks=chunks, data=np.random.random(shape)) + print(f"HDF5 '{filepath}': shape: {shape}, " f"chunks: {chunks}, dtype: {dtype}") + + +@contextlib.contextmanager +def dask_h5py(args): + import h5py + from dask import array as da + from dask_cuda import LocalCUDACluster + from distributed import Client + + def f(): + t0 = clock() + with h5py.File(args.dir / "A", "r") as af: + with h5py.File(args.dir / "B", "r") as bf: + a = da.from_array(af[DATASET], chunks=af[DATASET].chunks) + b = da.from_array(bf[DATASET], chunks=bf[DATASET].chunks) + c = args.op(da, a, b) + int(c.sum().compute()) + t1 = clock() + return t1 - t0 + + with LocalCUDACluster(n_workers=args.n_workers) as cluster: + with Client(cluster): + yield f + + +@contextlib.contextmanager +def run_legate(args): + import cunumeric as num + + from legate.core import get_legate_runtime + from legate_kvikio.kerchunk import hdf5_read + + def f(): + get_legate_runtime().issue_execution_fence(block=True) + t0 = clock() + a = hdf5_read(args.dir / "A", dataset_name=DATASET) + b = hdf5_read(args.dir / "B", dataset_name=DATASET) + c = args.op(num, a, b) + int(c.sum()) + t1 = clock() + return t1 - t0 + + yield f + + +API = { + "dask-h5py": dask_h5py, + "legate": run_legate, +} + +OP = {"add": lambda xp, a, b: a + b, "matmul": lambda xp, a, b: xp.matmul(a, b)} + + +def main(args): + create_hdf5_array(args.dir / "A", chunks=(args.c, args.c), shape=(args.m, args.m)) + create_hdf5_array(args.dir / "B", chunks=(args.c, args.c), shape=(args.m, args.m)) + + timings = [] + with API[args.api](args) as f: + for _ in range(args.n_warmup_runs): + elapsed = f() + print("elapsed[warmup]: ", elapsed) + for i in range(args.nruns): + elapsed = f() + print(f"elapsed[run #{i}]: ", elapsed) + timings.append(elapsed) + print(f"elapsed mean: {np.mean(timings):.5}s (std: {np.std(timings):.5}s)") + + +if __name__ == "__main__": + + def parse_directory(x): + if x is None: + return x + else: + p = pathlib.Path(x) + if not p.is_dir(): + raise argparse.ArgumentTypeError("Must be a directory") + return p + + parser = argparse.ArgumentParser(description="Matrix operation on two Zarr files") + parser.add_argument( + "-m", + default=100, + type=int, + help="Dimension of the two squired input matrix (MxM) (default: %(default)s).", + ) + parser.add_argument( + "-c", + default=None, + type=int, + help="Dimension of the squired chunk (CxC) (default: M/10).", + ) + parser.add_argument( + "-d", + "--dir", + metavar="PATH", + default=None, + type=parse_directory, + help="Path to the directory to r/w from (default: tempfile.TemporaryDirectory)", + ) + parser.add_argument( + "--nruns", + metavar="RUNS", + default=1, + type=int, + help="Number of runs (default: %(default)s).", + ) + parser.add_argument( + "--api", + metavar="API", + default=tuple(API.keys())[0], + choices=tuple(API.keys()), + help="API to use {%(choices)s}", + ) + parser.add_argument( + "--n-workers", + default=1, + type=int, + help="Number of workers (default: %(default)s).", + ) + parser.add_argument( + "--op", + metavar="OP", + default=tuple(OP.keys())[0], + choices=tuple(OP.keys()), + help="Operation to run {%(choices)s}", + ) + parser.add_argument( + "--n-warmup-runs", + default=1, + type=int, + help="Number of warmup runs (default: %(default)s).", + ) + + args = parser.parse_args() + args.op = OP[args.op] # Parse the operation argument + if args.c is None: + args.c = args.m // 10 + + # Create a temporary directory if user didn't specify a directory + temp_dir: tempfile.TemporaryDirectory | ContextManager + if args.dir is None: + temp_dir = tempfile.TemporaryDirectory() + args.dir = pathlib.Path(temp_dir.name) + else: + temp_dir = contextlib.nullcontext() + + with temp_dir: + main(args) diff --git a/legate/examples/hdf5_io.py b/legate/examples/hdf5_io.py index d8f0a876d9..5332cd571a 100644 --- a/legate/examples/hdf5_io.py +++ b/legate/examples/hdf5_io.py @@ -6,7 +6,6 @@ import numpy as np import legate_kvikio.kerchunk -import legate_kvikio.zarr def hdf5_io(filename): From 42d5a6e73f6f3b712fe061cc85a4930d805288c9 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 23 May 2023 13:53:57 +0200 Subject: [PATCH 06/16] benchmark --- legate/benchmarks/hdf5_read.py | 188 +++++++++++++++++++++++++++++++++ legate/examples/hdf5_io.py | 1 - 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 legate/benchmarks/hdf5_read.py diff --git a/legate/benchmarks/hdf5_read.py b/legate/benchmarks/hdf5_read.py new file mode 100644 index 0000000000..70821aa86b --- /dev/null +++ b/legate/benchmarks/hdf5_read.py @@ -0,0 +1,188 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + +import argparse +import contextlib +import pathlib +import tempfile +from time import perf_counter as clock +from typing import ContextManager + +import h5py +import numpy as np + +DATASET = "dataset-hdf5-read" + + +def try_open_hdf5_array(filepath, shape, chunks, dtype): + try: + with h5py.File(filepath, "r") as f: + a = f[DATASET] + if a.shape == shape and a.chunks == chunks and a.dtype == dtype: + return a + except BaseException: + pass + return None + + +def create_hdf5_array(filepath: pathlib.Path, shape, chunks, dtype=np.float64) -> None: + ret = try_open_hdf5_array(filepath, shape, chunks, dtype) + if ret is None: + filepath.unlink(missing_ok=True) + # Write array using h5py + with h5py.File(filepath, "w") as f: + f.create_dataset(DATASET, chunks=chunks, data=np.random.random(shape)) + print(f"HDF5 '{filepath}': shape: {shape}, " f"chunks: {chunks}, dtype: {dtype}") + + +@contextlib.contextmanager +def dask_h5py(args): + import cupy + import h5py + from dask import array as da + from dask_cuda import LocalCUDACluster + from distributed import Client + + def f(): + t0 = clock() + with h5py.File(args.dir / "A", "r") as af: + with h5py.File(args.dir / "B", "r") as bf: + a = da.from_array(af[DATASET], chunks=af[DATASET].chunks) + b = da.from_array(bf[DATASET], chunks=bf[DATASET].chunks) + a = a.map_blocks(cupy.asarray) + b = b.map_blocks(cupy.asarray) + c = args.op(da, a, b) + int(c.sum().compute()) + t1 = clock() + return t1 - t0 + + with LocalCUDACluster(n_workers=args.n_workers) as cluster: + with Client(cluster): + yield f + + +@contextlib.contextmanager +def run_legate(args): + import cunumeric as num + + from legate.core import get_legate_runtime + from legate_kvikio.kerchunk import hdf5_read + + def f(): + get_legate_runtime().issue_execution_fence(block=True) + t0 = clock() + a = hdf5_read(args.dir / "A", dataset_name=DATASET) + b = hdf5_read(args.dir / "B", dataset_name=DATASET) + c = args.op(num, a, b) + int(c.sum()) + t1 = clock() + return t1 - t0 + + yield f + + +API = { + "dask-h5py": dask_h5py, + "legate": run_legate, +} + +OP = {"add": lambda xp, a, b: a + b, "matmul": lambda xp, a, b: xp.matmul(a, b)} + + +def main(args): + create_hdf5_array(args.dir / "A", chunks=(args.c, args.c), shape=(args.m, args.m)) + create_hdf5_array(args.dir / "B", chunks=(args.c, args.c), shape=(args.m, args.m)) + + timings = [] + with API[args.api](args) as f: + for _ in range(args.n_warmup_runs): + elapsed = f() + print("elapsed[warmup]: ", elapsed) + for i in range(args.nruns): + elapsed = f() + print(f"elapsed[run #{i}]: ", elapsed) + timings.append(elapsed) + print(f"elapsed mean: {np.mean(timings):.5}s (std: {np.std(timings):.5}s)") + + +if __name__ == "__main__": + + def parse_directory(x): + if x is None: + return x + else: + p = pathlib.Path(x) + if not p.is_dir(): + raise argparse.ArgumentTypeError("Must be a directory") + return p + + parser = argparse.ArgumentParser(description="Matrix operation on two Zarr files") + parser.add_argument( + "-m", + default=100, + type=int, + help="Dimension of the two squired input matrix (MxM) (default: %(default)s).", + ) + parser.add_argument( + "-c", + default=None, + type=int, + help="Dimension of the squired chunk (CxC) (default: M/10).", + ) + parser.add_argument( + "-d", + "--dir", + metavar="PATH", + default=None, + type=parse_directory, + help="Path to the directory to r/w from (default: tempfile.TemporaryDirectory)", + ) + parser.add_argument( + "--nruns", + metavar="RUNS", + default=1, + type=int, + help="Number of runs (default: %(default)s).", + ) + parser.add_argument( + "--api", + metavar="API", + default=tuple(API.keys())[0], + choices=tuple(API.keys()), + help="API to use {%(choices)s}", + ) + parser.add_argument( + "--n-workers", + default=1, + type=int, + help="Number of workers (default: %(default)s).", + ) + parser.add_argument( + "--op", + metavar="OP", + default=tuple(OP.keys())[0], + choices=tuple(OP.keys()), + help="Operation to run {%(choices)s}", + ) + parser.add_argument( + "--n-warmup-runs", + default=1, + type=int, + help="Number of warmup runs (default: %(default)s).", + ) + + args = parser.parse_args() + args.op = OP[args.op] # Parse the operation argument + if args.c is None: + args.c = args.m // 10 + + # Create a temporary directory if user didn't specify a directory + temp_dir: tempfile.TemporaryDirectory | ContextManager + if args.dir is None: + temp_dir = tempfile.TemporaryDirectory() + args.dir = pathlib.Path(temp_dir.name) + else: + temp_dir = contextlib.nullcontext() + + with temp_dir: + main(args) diff --git a/legate/examples/hdf5_io.py b/legate/examples/hdf5_io.py index d8f0a876d9..5332cd571a 100644 --- a/legate/examples/hdf5_io.py +++ b/legate/examples/hdf5_io.py @@ -6,7 +6,6 @@ import numpy as np import legate_kvikio.kerchunk -import legate_kvikio.zarr def hdf5_io(filename): From 34f1db28b1e93ceb77d19695bff9efa16c5dcf57 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 08:29:14 +0200 Subject: [PATCH 07/16] Typo Co-authored-by: Lawrence Mitchell --- legate/benchmarks/hdf5_read.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legate/benchmarks/hdf5_read.py b/legate/benchmarks/hdf5_read.py index 70821aa86b..b11114cad9 100644 --- a/legate/benchmarks/hdf5_read.py +++ b/legate/benchmarks/hdf5_read.py @@ -121,13 +121,13 @@ def parse_directory(x): "-m", default=100, type=int, - help="Dimension of the two squired input matrix (MxM) (default: %(default)s).", + help="Dimension of the two square input matrices (MxM) (default: %(default)s).", ) parser.add_argument( "-c", default=None, type=int, - help="Dimension of the squired chunk (CxC) (default: M/10).", + help="Dimension of the square chunks (CxC) (default: M//10).", ) parser.add_argument( "-d", From 59823960983ef296592d795968be8eaf079a0077 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 08:27:53 +0200 Subject: [PATCH 08/16] doc & clean up --- legate/benchmarks/hdf5_read.py | 4 +++- legate/cpp/tile_io.cpp | 10 ++++---- legate/legate_kvikio/kerchunk.py | 9 ++++--- legate/legate_kvikio/tile.py | 41 ++++++++++++++++++++++++++------ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/legate/benchmarks/hdf5_read.py b/legate/benchmarks/hdf5_read.py index 9e096e8ff9..cd4837a98a 100644 --- a/legate/benchmarks/hdf5_read.py +++ b/legate/benchmarks/hdf5_read.py @@ -3,6 +3,7 @@ import argparse import contextlib +import operator import pathlib import tempfile from time import perf_counter as clock @@ -83,7 +84,8 @@ def f(): "legate": run_legate, } -OP = {"add": lambda xp, a, b: a + b, "matmul": lambda xp, a, b: xp.matmul(a, b)} + +OP = {"add": operator.add, "matmul": operator.matmul} def main(args): diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index 3986f613c2..3f53dc455b 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -119,11 +119,11 @@ size_t linearize(const legate::DomainPoint& lo_dp, const legate::DomainPoint& hi_dp, const legate::DomainPoint& point_dp) { - legate::Point lo = lo_dp; - legate::Point hi = hi_dp; - legate::Point point = point_dp; - legate::Point extents = hi - lo + legate::Point::ONES(); - size_t idx = 0; + const legate::Point lo = lo_dp; + const legate::Point hi = hi_dp; + const legate::Point point = point_dp; + const legate::Point extents = hi - lo + legate::Point::ONES(); + size_t idx = 0; for (int32_t dim = 0; dim < DIM; ++dim) { idx = idx * extents[dim] + point[dim] - lo[dim]; } diff --git a/legate/legate_kvikio/kerchunk.py b/legate/legate_kvikio/kerchunk.py index 8e51c86d28..b9df757b04 100644 --- a/legate/legate_kvikio/kerchunk.py +++ b/legate/legate_kvikio/kerchunk.py @@ -18,7 +18,10 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndarray: - """Read an HDF5 array from disk using KvikIO + """Read an HDF5 array from disk using Kerchunk and KvikIO + + We use Kerchunk's `SingleHdf5ToZarr` to find the data chunks embedded + in the hdf5 file. If it fails for any reason, this function fails as well. Notes ----- @@ -63,7 +66,7 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndar ret = cunumeric.empty(shape=zarr_ary.shape, dtype=zarr_ary.dtype) read_tiles_by_offsets( ret, - filepaths=[filepath], + filepath=filepath, offsets=tuple(offsets), sizes=tuple(sizes), tile_shape=zarr_ary.chunks, @@ -71,7 +74,7 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndar else: read_tiles_by_offsets( padded_ary, - filepaths=[filepath], + filepath=filepath, offsets=tuple(offsets), sizes=tuple(sizes), tile_shape=zarr_ary.chunks, diff --git a/legate/legate_kvikio/tile.py b/legate/legate_kvikio/tile.py index 6d63af9043..5c46ee2bf5 100644 --- a/legate/legate_kvikio/tile.py +++ b/legate/legate_kvikio/tile.py @@ -106,14 +106,30 @@ def read_tiles( def read_tiles_by_offsets( ary: cunumeric.ndarray, - filepaths: Iterable[pathlib.Path | str], + filepath: Iterable[pathlib.Path | str], offsets: Tuple[int], sizes: Tuple[int], tile_shape: Tuple[int], ) -> None: - # TODO; support a filepath per offset/size - assert len(filepaths) == 1, "For now, only a single filepath is supported" - filepath = str(filepaths[0]) + """Read multiple tiles from a single file into an array using KvikIO + + The array shape must be divisible with the tile shape. + + # TODO: support a filepath per offset/size + + Parameters + ---------- + ary + The cuNumeric array to read into. + filepath + Filepath to the file. + offsets + The offset of each tile in the file (in bytes). + sizes + The size of each tile in the file (in bytes). + tile_shape + The shape of each tile. + """ if any(d % c != 0 for d, c in zip(ary.shape, tile_shape)): raise ValueError( @@ -126,14 +142,25 @@ def read_tiles_by_offsets( # Use the partition's color shape as the launch shape so there will be # one task for each tile - launch_shape = store_partition.partition.color_shape + launch_shape = Rect(store_partition.partition.color_shape) + launch_vol = launch_shape.get_volume() + if launch_vol != len(offsets): + raise ValueError( + f"Number of offsets ({len(offsets)}) must match the number " + f"of tiles of `ary` ({launch_vol})" + ) + if launch_vol != len(offsets): + raise ValueError( + f"Number of sizes ({len(sizes)}) must match the number " + f"of tiles of `ary` ({launch_vol})" + ) task = context.create_manual_task( TaskOpCode.TILE_READ_BY_OFFSETS, - launch_domain=Rect(launch_shape), + launch_domain=launch_shape, ) task.add_output(store_partition) - task.add_scalar_arg(filepath, types.string) + task.add_scalar_arg(str(filepath), types.string) task.add_scalar_arg(offsets, (types.uint64,)) task.add_scalar_arg(sizes, (types.uint64,)) task.add_scalar_arg(tile_shape, (types.uint64,)) From 66bebedb3e6ac37eb69904bd4dcef7bfadf1c382 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 09:14:05 +0200 Subject: [PATCH 09/16] linearize(): zero-origin up front --- legate/cpp/tile_io.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index 3f53dc455b..904ac4a343 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -121,11 +121,11 @@ size_t linearize(const legate::DomainPoint& lo_dp, { const legate::Point lo = lo_dp; const legate::Point hi = hi_dp; - const legate::Point point = point_dp; + const legate::Point point = point_dp - lo_dp; const legate::Point extents = hi - lo + legate::Point::ONES(); size_t idx = 0; for (int32_t dim = 0; dim < DIM; ++dim) { - idx = idx * extents[dim] + point[dim] - lo[dim]; + idx = idx * extents[dim] + point[dim]; } return idx; } From 36c481048876d8c8018de8f04a63e3bc19bd1ff5 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 15:33:21 +0200 Subject: [PATCH 10/16] Typo Co-authored-by: Lawrence Mitchell --- legate/legate_kvikio/tile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legate/legate_kvikio/tile.py b/legate/legate_kvikio/tile.py index 5c46ee2bf5..68372355eb 100644 --- a/legate/legate_kvikio/tile.py +++ b/legate/legate_kvikio/tile.py @@ -149,7 +149,7 @@ def read_tiles_by_offsets( f"Number of offsets ({len(offsets)}) must match the number " f"of tiles of `ary` ({launch_vol})" ) - if launch_vol != len(offsets): + if launch_vol != len(sizes): raise ValueError( f"Number of sizes ({len(sizes)}) must match the number " f"of tiles of `ary` ({launch_vol})" From a0e14f919c9407d59d03014debfdd689d112e557 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 15:42:42 +0200 Subject: [PATCH 11/16] removed the sizes argument --- legate/cpp/tile_io.cpp | 7 +------ legate/legate_kvikio/kerchunk.py | 6 ++---- legate/legate_kvikio/tile.py | 9 --------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index 904ac4a343..db5ac3446a 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -145,8 +145,7 @@ struct tile_read_by_offsets_fn { const auto launch_domain = context.get_launch_domain(); const std::string path = context.scalars().at(0).value(); legate::Span offsets = context.scalars().at(1).values(); - legate::Span sizes = context.scalars().at(2).values(); - legate::Span tile_shape = context.scalars().at(3).values(); + legate::Span tile_shape = context.scalars().at(2).values(); // Flatten task index uint32_t flatten_task_index = 0; @@ -159,9 +158,6 @@ struct tile_read_by_offsets_fn { if (shape_volume == 0) { return; } size_t nbytes = shape_volume * sizeof(DTYPE); std::array strides{}; - if (nbytes != sizes[flatten_task_index]) { - throw std::runtime_error("sizes doesn't match the size of the output store"); - } // We know that the accessor is contiguous because we set `policy.exact = true` // in `Mapper::store_mappings()`. @@ -239,7 +235,6 @@ class TileReadTask : public Task { * - scalars: * - path: std::string * - offsets: tuple of int64_t - * - sizes: tuple of int64_t * - tile_shape: tuple of int64_t * - outputs: * - buffer: store (any dtype) diff --git a/legate/legate_kvikio/kerchunk.py b/legate/legate_kvikio/kerchunk.py index b9df757b04..473e03d523 100644 --- a/legate/legate_kvikio/kerchunk.py +++ b/legate/legate_kvikio/kerchunk.py @@ -52,14 +52,14 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndar # Extract offset and bytes for each chunk refs = annotations["refs"] offsets = [] - sizes = [] + tile_nbytes = math.prod(zarr_ary.chunks) * zarr_ary.itemsize for chunk_coord in itertools.product( *(range(math.ceil(s / c)) for s, c in zip(zarr_ary.shape, zarr_ary.chunks)) ): key = zarr_ary._chunk_key(chunk_coord) _, offset, nbytes = refs[key] offsets.append(offset) - sizes.append(nbytes) + assert tile_nbytes == nbytes padded_ary = get_padded_array(zarr_ary) if padded_ary is None: @@ -68,7 +68,6 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndar ret, filepath=filepath, offsets=tuple(offsets), - sizes=tuple(sizes), tile_shape=zarr_ary.chunks, ) else: @@ -76,7 +75,6 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndar padded_ary, filepath=filepath, offsets=tuple(offsets), - sizes=tuple(sizes), tile_shape=zarr_ary.chunks, ) ret = padded_ary[tuple(slice(s) for s in zarr_ary.shape)] diff --git a/legate/legate_kvikio/tile.py b/legate/legate_kvikio/tile.py index 68372355eb..6f2e73d076 100644 --- a/legate/legate_kvikio/tile.py +++ b/legate/legate_kvikio/tile.py @@ -108,7 +108,6 @@ def read_tiles_by_offsets( ary: cunumeric.ndarray, filepath: Iterable[pathlib.Path | str], offsets: Tuple[int], - sizes: Tuple[int], tile_shape: Tuple[int], ) -> None: """Read multiple tiles from a single file into an array using KvikIO @@ -125,8 +124,6 @@ def read_tiles_by_offsets( Filepath to the file. offsets The offset of each tile in the file (in bytes). - sizes - The size of each tile in the file (in bytes). tile_shape The shape of each tile. """ @@ -149,11 +146,6 @@ def read_tiles_by_offsets( f"Number of offsets ({len(offsets)}) must match the number " f"of tiles of `ary` ({launch_vol})" ) - if launch_vol != len(sizes): - raise ValueError( - f"Number of sizes ({len(sizes)}) must match the number " - f"of tiles of `ary` ({launch_vol})" - ) task = context.create_manual_task( TaskOpCode.TILE_READ_BY_OFFSETS, launch_domain=launch_shape, @@ -162,6 +154,5 @@ def read_tiles_by_offsets( task.add_output(store_partition) task.add_scalar_arg(str(filepath), types.string) task.add_scalar_arg(offsets, (types.uint64,)) - task.add_scalar_arg(sizes, (types.uint64,)) task.add_scalar_arg(tile_shape, (types.uint64,)) task.execute() From 5ad1697e7d30f128bcbb3529a04d3ed03d687e53 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 15:53:42 +0200 Subject: [PATCH 12/16] removed strides --- legate/cpp/tile_io.cpp | 5 ++--- legate/legate_kvikio/tile.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index db5ac3446a..c8e6f41881 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -90,17 +90,16 @@ struct tile_read_write_fn { auto shape_volume = shape.volume(); if (shape_volume == 0) { return; } size_t nbytes = shape_volume * sizeof(DTYPE); - std::array strides{}; // We know that the accessor is contiguous because we set `policy.exact = true` // in `Mapper::store_mappings()`. if constexpr (IsReadOperation) { kvikio::FileHandle f(filepath, "r"); - auto* data = store.write_accessor().ptr(shape, strides.data()); + auto* data = store.write_accessor().ptr(shape); f.pread(data, nbytes).get(); } else { kvikio::FileHandle f(filepath, "w"); - const auto* data = store.read_accessor().ptr(shape, strides.data()); + const auto* data = store.read_accessor().ptr(shape); f.pwrite(data, nbytes).get(); } } diff --git a/legate/legate_kvikio/tile.py b/legate/legate_kvikio/tile.py index 6f2e73d076..395b2f6f55 100644 --- a/legate/legate_kvikio/tile.py +++ b/legate/legate_kvikio/tile.py @@ -21,7 +21,7 @@ def _tile_read_write( dirpath: pathlib.Path | str, tile_shape: Tuple[int], tile_start: Optional[Tuple[int]], -): +) -> None: """Implementation of `write_tiles` and `read_tiles`""" dirpath = pathlib.Path(dirpath) @@ -77,7 +77,7 @@ def write_tiles( tile_start The start coordinate of the tiles """ - return _tile_read_write(TaskOpCode.TILE_WRITE, ary, dirpath, tile_shape, tile_start) + _tile_read_write(TaskOpCode.TILE_WRITE, ary, dirpath, tile_shape, tile_start) def read_tiles( @@ -101,7 +101,7 @@ def read_tiles( tile_start The start coordinate of the tiles """ - return _tile_read_write(TaskOpCode.TILE_READ, ary, dirpath, tile_shape, tile_start) + _tile_read_write(TaskOpCode.TILE_READ, ary, dirpath, tile_shape, tile_start) def read_tiles_by_offsets( From 3b235bb8ba3dae90514e653890c7596ce330a368 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 15:57:59 +0200 Subject: [PATCH 13/16] removed single-task path --- legate/cpp/tile_io.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index c8e6f41881..f4c951a92c 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -147,10 +147,8 @@ struct tile_read_by_offsets_fn { legate::Span tile_shape = context.scalars().at(2).values(); // Flatten task index - uint32_t flatten_task_index = 0; - if (!context.is_single_task()) { - flatten_task_index = linearize(launch_domain.lo(), launch_domain.hi(), task_index); - } + uint32_t flatten_task_index = + linearize(launch_domain.lo(), launch_domain.hi(), task_index); auto shape = store.shape(); auto shape_volume = shape.volume(); From b9001be9261844940f7adacc33407614bedd96a6 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 15:59:55 +0200 Subject: [PATCH 14/16] tile-shape: check number of dimensions --- legate/legate_kvikio/tile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/legate/legate_kvikio/tile.py b/legate/legate_kvikio/tile.py index 395b2f6f55..54f9fba7cf 100644 --- a/legate/legate_kvikio/tile.py +++ b/legate/legate_kvikio/tile.py @@ -28,6 +28,8 @@ def _tile_read_write( if tile_start is None: tile_start = (0,) * len(tile_shape) + if len(ary.shape) != len(tile_shape): + raise ValueError("Tile shape and array shape must have same number of axes") if any(d % c != 0 for d, c in zip(ary.shape, tile_shape)): raise ValueError( f"The tile shape {tile_shape} must be " @@ -128,6 +130,8 @@ def read_tiles_by_offsets( The shape of each tile. """ + if len(ary.shape) != len(tile_shape): + raise ValueError("Tile shape and array shape must have same number of axes") if any(d % c != 0 for d, c in zip(ary.shape, tile_shape)): raise ValueError( f"The tile shape {tile_shape} must be " From f5549df637e35815d0bc1897c19817e597530e71 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 16:04:46 +0200 Subject: [PATCH 15/16] doc --- legate/legate_kvikio/kerchunk.py | 5 ++--- legate/legate_kvikio/zarr.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/legate/legate_kvikio/kerchunk.py b/legate/legate_kvikio/kerchunk.py index 473e03d523..a226fe965e 100644 --- a/legate/legate_kvikio/kerchunk.py +++ b/legate/legate_kvikio/kerchunk.py @@ -25,9 +25,8 @@ def hdf5_read(filepath: pathlib.Path | str, dataset_name: str) -> cunumeric.ndar Notes ----- - The returned array is padded to make its shape divisible by the shape of - the Zarr chunks on disk (if not already). This means that the returned - Legate store can be larger than the returned cuNumeric array. + The returned array might be a view of a underlying array that has been padded in + order to make its shape divisible by the shape of the Zarr chunks on disk. Parameters ---------- diff --git a/legate/legate_kvikio/zarr.py b/legate/legate_kvikio/zarr.py index f4e0052b23..03a466448c 100644 --- a/legate/legate_kvikio/zarr.py +++ b/legate/legate_kvikio/zarr.py @@ -87,9 +87,8 @@ def read_array(dirpath: pathlib.Path | str) -> cunumeric.ndarray: Notes ----- - The returned array is padded to make its shape divisible by the shape of - the Zarr chunks on disk (if not already). This means that the returned - Legate store can be larger than the returned cuNumeric array. + The returned array might be a view of a underlying array that has been padded in + order to make its shape divisible by the shape of the Zarr chunks on disk. Parameters ---------- From f23b8be1f8693dc096e3a03b6aeb813ae74c250d Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Tue, 30 May 2023 16:59:01 +0200 Subject: [PATCH 16/16] removed more strides --- legate/cpp/tile_io.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/legate/cpp/tile_io.cpp b/legate/cpp/tile_io.cpp index f4c951a92c..3ef29704ae 100644 --- a/legate/cpp/tile_io.cpp +++ b/legate/cpp/tile_io.cpp @@ -154,12 +154,11 @@ struct tile_read_by_offsets_fn { auto shape_volume = shape.volume(); if (shape_volume == 0) { return; } size_t nbytes = shape_volume * sizeof(DTYPE); - std::array strides{}; // We know that the accessor is contiguous because we set `policy.exact = true` // in `Mapper::store_mappings()`. kvikio::FileHandle f(path, "r"); - auto* data = store.write_accessor().ptr(shape, strides.data()); + auto* data = store.write_accessor().ptr(shape); f.pread(data, nbytes, offsets[flatten_task_index]).get(); } };