diff --git a/Makefile b/Makefile index bfe9fea3b4..5066cc0244 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ ARKOUDA_PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) PROJECT_NAME := arkouda ARKOUDA_SOURCE_DIR := $(ARKOUDA_PROJECT_DIR)/src +ARKOUDA_REGISTRY_DIR=$(ARKOUDA_SOURCE_DIR)/registry ARKOUDA_MAIN_MODULE := arkouda_server ARKOUDA_MAKEFILES := Makefile Makefile.paths @@ -85,6 +86,10 @@ ifndef ARKOUDA_CONFIG_FILE ARKOUDA_CONFIG_FILE := $(ARKOUDA_PROJECT_DIR)/ServerModules.cfg endif +ifndef ARKOUDA_REGISTRATION_CONFIG +ARKOUDA_REGISTRATION_CONFIG := $(ARKOUDA_PROJECT_DIR)/registration-config.json +endif + CHPL_FLAGS += -lhdf5 -lhdf5_hl -lzmq -liconv -lidn2 -lparquet -larrow ARROW_READ_FILE_NAME += $(ARKOUDA_SOURCE_DIR)/parquet/ReadParquet @@ -380,9 +385,9 @@ endif SERVER_CONFIG_SCRIPT=$(ARKOUDA_SOURCE_DIR)/parseServerConfig.py # This is the main compilation statement section $(ARKOUDA_MAIN_MODULE): check-deps $(ARROW_UTIL_O) $(ARROW_READ_O) $(ARROW_WRITE_O) $(ARKOUDA_SOURCES) $(ARKOUDA_MAKEFILES) - $(eval MOD_GEN_OUT=$(shell python3 $(SERVER_CONFIG_SCRIPT) $(ARKOUDA_CONFIG_FILE) $(ARKOUDA_SOURCE_DIR))) + $(eval MOD_GEN_OUT=$(shell python3 $(SERVER_CONFIG_SCRIPT) $(ARKOUDA_CONFIG_FILE) $(ARKOUDA_REGISTRATION_CONFIG) $(ARKOUDA_SOURCE_DIR))) - $(CHPL) $(CHPL_DEBUG_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(CHPL_COMPAT_FLAGS) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) $(ARKOUDA_SERVER_USER_MODULES) $(MOD_GEN_OUT) $(ARKOUDA_RW_DEFAULT_FLAG) -I$(ARKOUDA_SOURCE_DIR)/parquet -o $@ + $(CHPL) $(CHPL_DEBUG_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(CHPL_COMPAT_FLAGS) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) $(ARKOUDA_SERVER_USER_MODULES) $(MOD_GEN_OUT) $(ARKOUDA_RW_DEFAULT_FLAG) $(ARKOUDA_REGISTRY_DIR)/Commands.chpl -I$(ARKOUDA_SOURCE_DIR)/parquet -o $@ CLEAN_TARGETS += arkouda-clean .PHONY: arkouda-clean @@ -568,6 +573,14 @@ benchmark: version: @echo $(VERSION); + +##################### +#### register.mk #### +##################### + +register-commands: + python3 $(ARKOUDA_REGISTRY_DIR)/register_commands.py $(ARKOUDA_REGISTRATION_CONFIG) $(ARKOUDA_CONFIG_FILE) $(ARKOUDA_SOURCE_DIR) + ##################### #### Epilogue.mk #### ##################### diff --git a/PROTO_tests/tests/client_test.py b/PROTO_tests/tests/client_test.py index 6b158e2834..9df8ee5759 100644 --- a/PROTO_tests/tests/client_test.py +++ b/PROTO_tests/tests/client_test.py @@ -137,7 +137,7 @@ def test_client_get_server_commands(self): sample of commands. """ cmds = ak.client.get_server_commands() - for cmd in ["connect", "array", "create1D", "tondarray1D", "info", "str"]: + for cmd in ["connect", "create1D", "tondarray1D", "info", "str"]: assert cmd in cmds def test_client_array_dim_cmd_error(self): diff --git a/arkouda/array_api/_typing.py b/arkouda/array_api/_typing.py index be6f86de80..78497d04ee 100644 --- a/arkouda/array_api/_typing.py +++ b/arkouda/array_api/_typing.py @@ -44,11 +44,9 @@ class NestedSequence(Protocol[_T_co]): - def __getitem__(self, key: int, /) -> _T_co | NestedSequence[_T_co]: - ... + def __getitem__(self, key: int, /) -> _T_co | NestedSequence[_T_co]: ... - def __len__(self, /) -> int: - ... + def __len__(self, /) -> int: ... Device = Literal["cpu"] @@ -75,5 +73,4 @@ def __len__(self, /) -> int: class SupportsDLPack(Protocol): - def __dlpack__(self, /, *, stream: None = ...) -> PyCapsule: - ... + def __dlpack__(self, /, *, stream: None = ...) -> PyCapsule: ... diff --git a/arkouda/array_api/array_object.py b/arkouda/array_api/array_object.py index 6c03c5e552..5b642314c0 100644 --- a/arkouda/array_api/array_object.py +++ b/arkouda/array_api/array_object.py @@ -122,7 +122,9 @@ def item(self): if self._has_single_elem(): return self._array[0] else: - raise ValueError("Can only convert an array with one element to a Python scalar") + raise ValueError( + "Can only convert an array with one element to a Python scalar" + ) def transpose(self, axes: Optional[Tuple[int, ...]] = None): """ @@ -131,6 +133,7 @@ def transpose(self, axes: Optional[Tuple[int, ...]] = None): For axes=None, reverse all the dimensions of the array. """ from .manipulation_functions import permute_dims + if axes is None: _axes = tuple(range(self.ndim - 1, -1, -1)) else: @@ -185,7 +188,9 @@ def chunk_info(self: Array, /) -> List[List[int]]: """ import json - return json.loads(ak.generic_msg(cmd=f"chunkInfo{self.ndim}D", args={"array": self._array})) + return json.loads( + ak.generic_msg(cmd=f"chunkInfo{self.ndim}D", args={"array": self._array}) + ) def __array__(self, dtype: None | np.dtype[Any] = None): """ @@ -459,7 +464,7 @@ def __getitem__( if isinstance(kt, Array): if kt.size == 1 or kt.shape == (): k.append(kt._array[0]) - else : + else: k.append(kt._array) elif isinstance(kt, np.ndarray): k.append(asarray(kt)._array) @@ -504,7 +509,9 @@ def __index__(self: Array, /) -> int: if isinstance(s, int): return s else: - raise TypeError("Only integer arrays can be converted to a Python integer") + raise TypeError( + "Only integer arrays can be converted to a Python integer" + ) else: raise ValueError("cannot convert non-scalar array to int") @@ -607,9 +614,9 @@ def __pow__(self: Array, other: Union[int, float, Array], /) -> Array: Compute the power of this array by another array or scalar. """ if isinstance(other, (int, float)): - return Array._new(self._array ** other) + return Array._new(self._array**other) else: - return Array._new(self._array ** other._array) + return Array._new(self._array**other._array) def __rshift__(self: Array, other: Union[int, Array], /) -> Array: """ @@ -842,9 +849,9 @@ def __rpow__(self: Array, other: Union[int, float, Array], /) -> Array: Compute the power of another array or scalar by this array. """ if isinstance(other, (int, float)): - return Array._new(other ** self._array) + return Array._new(other**self._array) else: - return Array._new(other._array ** self._array) + return Array._new(other._array**self._array) def __irshift__(self: Array, other: Union[int, Array], /) -> Array: """ @@ -972,7 +979,9 @@ def T(self) -> Array: def implements_numpy(numpy_function): """Register an __array_function__ implementation for MyArray objects.""" + def decorator(func): HANDLED_FUNCTIONS[numpy_function] = func return func + return decorator diff --git a/arkouda/array_api/creation_functions.py b/arkouda/array_api/creation_functions.py index 06380292b6..e9b20f5753 100644 --- a/arkouda/array_api/creation_functions.py +++ b/arkouda/array_api/creation_functions.py @@ -82,7 +82,9 @@ def asarray( elif isinstance(obj, np.ndarray): return Array._new(_to_pdarray(obj, dt=dtype)) else: - raise ValueError("asarray not implemented for 'NestedSequence' or 'SupportsBufferProtocol'") + raise ValueError( + "asarray not implemented for 'NestedSequence' or 'SupportsBufferProtocol'" + ) def arange( diff --git a/arkouda/array_api/manipulation_functions.py b/arkouda/array_api/manipulation_functions.py index 8b9adcfc65..b9688fc0e0 100644 --- a/arkouda/array_api/manipulation_functions.py +++ b/arkouda/array_api/manipulation_functions.py @@ -4,8 +4,8 @@ from typing import List, Optional, Tuple, Union, cast from arkouda.client import generic_msg -from arkouda.pdarrayclass import create_pdarray -from arkouda.pdarraycreation import scalar_array +from arkouda.pdarrayclass import create_pdarray, create_pdarrays +from arkouda.pdarraycreation import scalar_array, promote_to_common_dtype from arkouda.util import broadcast_dims import numpy as np @@ -39,7 +39,7 @@ def broadcast_to(x: Array, /, shape: Tuple[int, ...]) -> Array: cast( str, generic_msg( - cmd=f"broadcastTo{x.ndim}Dx{len(shape)}D", + cmd=f"broadcast<{x.dtype},{x.ndim},{len(shape)}>", args={ "name": x._array, "shape": shape, @@ -67,19 +67,28 @@ def concat( flattened before concatenation. """ - # TODO: type promotion across input arrays + ndim = arrays[0].ndim + for a in arrays: + if a.ndim != ndim: + raise ValueError( + "all input arrays must have the same number of dimensions to concatenate" + ) + + (common_dt, _arrays) = promote_to_common_dtype([a._array for a in arrays]) return Array._new( create_pdarray( cast( str, generic_msg( - cmd=f"concat{arrays[0].ndim}D" - if axis is not None - else f"concatFlat{arrays[0].ndim}D", + cmd=( + f"concat<{common_dt},{ndim}>" + if axis is not None + else f"concatFlat<{common_dt},{ndim}>" + ), args={ "n": len(arrays), - "names": [a._array for a in arrays], + "names": _arrays, "axis": axis, }, ), @@ -106,7 +115,7 @@ def expand_dims(x: Array, /, *, axis: int) -> Array: cast( str, generic_msg( - cmd=f"expandDims{x.ndim}D", + cmd=f"expandDims<{x.dtype},{x.ndim}>", args={ "name": x._array, "axis": axis, @@ -139,7 +148,11 @@ def flip(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None) -> cast( str, generic_msg( - cmd=f"flipAll{x.ndim}D" if axis is None else f"flip{x.ndim}D", + cmd=( + f"flipAll<{x.dtype},{x.ndim}>" + if axis is None + else f"flip<{x.dtype},{x.ndim}>" + ), args={ "name": x._array, "nAxes": len(axisList), @@ -154,7 +167,10 @@ def flip(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None) -> def moveaxis( - x: Array, source: Union[int, Tuple[int, ...]], destination: Union[int, Tuple[int, ...]], / + x: Array, + source: Union[int, Tuple[int, ...]], + destination: Union[int, Tuple[int, ...]], + /, ) -> Array: """ Move axes of an array to new positions. @@ -176,11 +192,15 @@ def moveaxis( for s, d in zip(source, destination): perm[s] = d else: - raise ValueError("source and destination must both be tuples if source is a tuple") + raise ValueError( + "source and destination must both be tuples if source is a tuple" + ) elif isinstance(destination, int): perm[source] = destination else: - raise ValueError("source and destination must both be integers if source is a tuple") + raise ValueError( + "source and destination must both be integers if source is a tuple" + ) return permute_dims(x, axes=tuple(perm)) @@ -202,7 +222,7 @@ def permute_dims(x: Array, /, axes: Tuple[int, ...]) -> Array: cast( str, generic_msg( - cmd=f"permuteDims{x.ndim}D", + cmd=f"permuteDims<{x.dtype},{x.ndim}>", args={ "name": x._array, "axes": axes, @@ -215,7 +235,9 @@ def permute_dims(x: Array, /, axes: Tuple[int, ...]) -> Array: raise IndexError(f"Failed to permute array dimensions: {e}") -def repeat(x: Array, repeats: Union[int, Array], /, *, axis: Optional[int] = None) -> Array: +def repeat( + x: Array, repeats: Union[int, Array], /, *, axis: Optional[int] = None +) -> Array: """ Repeat elements of an array. @@ -242,7 +264,7 @@ def repeat(x: Array, repeats: Union[int, Array], /, *, axis: Optional[int] = Non cast( str, generic_msg( - cmd=f"repeatFlat{x.ndim}D", + cmd=f"repeatFlat<{x.dtype},{x.ndim}>", args={ "name": x._array, "repeats": reps._array, @@ -279,7 +301,7 @@ def reshape( cast( str, generic_msg( - cmd=f"reshape{x.ndim}Dx{len(shape)}D", + cmd=f"reshape<{x.dtype},{x.ndim},{len(shape)}>", args={ "name": x._array, "shape": shape, @@ -325,15 +347,17 @@ def roll( cast( str, generic_msg( - cmd=f"rollFlattened{x.ndim}D" - if axis is None - else f"roll{x.ndim}D", + cmd=( + f"rollFlattened<{x.dtype},{x.ndim}>" + if axis is None + else f"roll<{x.dtype},{x.ndim}>" + ), args={ "name": x._array, "nShifts": len(shift) if isinstance(shift, tuple) else 1, - "shift": list(shift) - if isinstance(shift, tuple) - else [shift], + "shift": ( + list(shift) if isinstance(shift, tuple) else [shift] + ), "nAxes": len(axisList), "axis": axisList, }, @@ -363,7 +387,7 @@ def squeeze(x: Array, /, axis: Union[int, Tuple[int, ...]]) -> Array: cast( str, generic_msg( - cmd=f"squeeze{x.ndim}Dx{x.ndim - nAxes}D", + cmd=f"squeeze<{x.dtype},{x.ndim},{x.ndim - nAxes}>", args={ "name": x._array, "nAxes": nAxes, @@ -393,15 +417,24 @@ def stack(arrays: Union[Tuple[Array, ...], List[Array]], /, *, axis: int = 0) -> of dimensions in the input arrays. The default is 0. """ + ndim = arrays[0].ndim + for a in arrays: + if a.ndim != ndim: + raise ValueError( + "all input arrays must have the same number of dimensions to stack" + ) + + (common_dt, _arrays) = promote_to_common_dtype([a._array for a in arrays]) + # TODO: type promotion across input arrays return Array._new( create_pdarray( cast( str, generic_msg( - cmd=f"stack{arrays[0].ndim}D", + cmd=f"stack<{common_dt},{ndim}>", args={ - "names": [a._array for a in arrays], + "names": _arrays, "n": len(arrays), "axis": axis, }, @@ -440,7 +473,7 @@ def tile(x: Array, repetitions: Tuple[int, ...], /) -> Array: cast( str, generic_msg( - cmd=f"tile{xr.ndim}D", + cmd=f"tile<{xr.dtype},{xr.ndim}>", args={ "name": xr._array, "reps": reps, @@ -462,16 +495,20 @@ def unstack(x: Array, /, *, axis: int = 0) -> Tuple[Array, ...]: axis : int, optional The axis along which to unstack the array. The default is 0. """ - resp = cast( - str, - generic_msg( - cmd=f"unstack{x.ndim}D", - args={ - "name": x._array, - "axis": axis, - "numReturnArrays": x.shape[axis], - }, - ), + return tuple( + Array._new( + create_pdarrays( + cast( + str, + generic_msg( + cmd=f"unstack<{x.dtype},{x.ndim}>", + args={ + "name": x._array, + "axis": axis, + "numReturnArrays": x.shape[axis], + }, + ), + ) ) - - return tuple([Array._new(create_pdarray(a)) for a in resp.split("+")]) + ) + ) diff --git a/arkouda/array_api/searching_functions.py b/arkouda/array_api/searching_functions.py index 4270719f22..282d4e75b8 100644 --- a/arkouda/array_api/searching_functions.py +++ b/arkouda/array_api/searching_functions.py @@ -112,7 +112,7 @@ def nonzero(x: Array, /) -> Tuple[Array, ...]: generic_msg( cmd=f"nonzero{x.ndim}D", args={"x": x._array}, - ) + ), ) return tuple([Array._new(create_pdarray(a)) for a in resp.split("+")]) @@ -149,7 +149,12 @@ def where(condition: Array, x1: Array, x2: Array, /) -> Array: def searchsorted( - x1: Array, x2: Array, /, *, side: Literal['left', 'right'] = 'left', sorter: Optional[Array] = None + x1: Array, + x2: Array, + /, + *, + side: Literal["left", "right"] = "left", + sorter: Optional[Array] = None, ) -> Array: """ Given a sorted array `x1`, find the indices to insert elements from another array `x2` such that diff --git a/arkouda/array_api/set_functions.py b/arkouda/array_api/set_functions.py index db3edfcafa..34034846f3 100644 --- a/arkouda/array_api/set_functions.py +++ b/arkouda/array_api/set_functions.py @@ -34,14 +34,14 @@ def unique_all(x: Array, /) -> UniqueAllResult: - the counts of each unique value """ resp = cast( - str, - generic_msg( - cmd=f"uniqueAll{x.ndim}D", - args={"name": x._array}, - ), - ) + str, + generic_msg( + cmd=f"uniqueAll{x.ndim}D", + args={"name": x._array}, + ), + ) - arrays = [Array._new(create_pdarray(r)) for r in resp.split('+')] + arrays = [Array._new(create_pdarray(r)) for r in resp.split("+")] return UniqueAllResult( values=arrays[0], @@ -58,14 +58,14 @@ def unique_counts(x: Array, /) -> UniqueCountsResult: - the counts of each unique value """ resp = cast( - str, - generic_msg( - cmd=f"uniqueCounts{x.ndim}D", - args={"name": x._array}, - ), - ) + str, + generic_msg( + cmd=f"uniqueCounts{x.ndim}D", + args={"name": x._array}, + ), + ) - arrays = [Array._new(create_pdarray(r)) for r in resp.split('+')] + arrays = [Array._new(create_pdarray(r)) for r in resp.split("+")] return UniqueCountsResult( values=arrays[0], @@ -80,14 +80,14 @@ def unique_inverse(x: Array, /) -> UniqueInverseResult: - the inverse indices that reconstruct `x` from the unique values """ resp = cast( - str, - generic_msg( - cmd=f"uniqueInverse{x.ndim}D", - args={"name": x._array}, - ), - ) + str, + generic_msg( + cmd=f"uniqueInverse{x.ndim}D", + args={"name": x._array}, + ), + ) - arrays = [Array._new(create_pdarray(r)) for r in resp.split('+')] + arrays = [Array._new(create_pdarray(r)) for r in resp.split("+")] return UniqueInverseResult( values=arrays[0], diff --git a/arkouda/array_api/statistical_functions.py b/arkouda/array_api/statistical_functions.py index 02e7858c55..2a25b7b30d 100644 --- a/arkouda/array_api/statistical_functions.py +++ b/arkouda/array_api/statistical_functions.py @@ -107,31 +107,28 @@ def mean( if x.dtype not in _real_floating_dtypes: raise TypeError("Only real floating-point dtypes are allowed in mean") - axis_list = [] if axis is not None: axis_list = list(axis) if isinstance(axis, tuple) else [axis] - - resp = generic_msg( - cmd=f"stats{x.ndim}D", - args={ - "x": x._array, - "comp": "mean", - "nAxes": len(axis_list), - "axis": axis_list, - "ddof": 0, - "skipNan": True, # TODO: handle all-nan slices - }, + else: + axis_list = list(range(x.ndim)) + + arr = Array._new( + create_pdarray( + generic_msg( + cmd=f"meanReduce<{x.dtype},{x.ndim}>", + args={ + "x": x._array, + "axes": axis_list, + "skipNan": True, # TODO: handle all-nan slices + }, + ) + ) ) - if axis is None or x.ndim == 1: - return Array._new(scalar_array(parse_single_value(resp))) + if keepdims or axis is None: + return arr else: - arr = Array._new(create_pdarray(resp)) - - if keepdims: - return arr - else: - return squeeze(arr, axis) + return squeeze(arr, axis) def min( @@ -272,31 +269,29 @@ def std( if correction < 0: raise ValueError("Correction must be non-negative in std") - axis_list = [] if axis is not None: axis_list = list(axis) if isinstance(axis, tuple) else [axis] - - resp = generic_msg( - cmd=f"stats{x.ndim}D", - args={ - "x": x._array, - "comp": "std", - "ddof": correction, - "nAxes": len(axis_list), - "axis": axis_list, - "skipNan": True, - }, + else: + axis_list = list(range(x.ndim)) + + arr = Array._new( + create_pdarray( + generic_msg( + cmd=f"stdReduce<{x.dtype},{x.ndim}>", + args={ + "x": x._array, + "ddof": correction, + "axes": axis_list, + "skipNan": True, + }, + ) + ) ) - if axis is None or x.ndim == 1: - return Array._new(scalar_array(parse_single_value(resp))) + if keepdims or axis is None: + return arr else: - arr = Array._new(create_pdarray(resp)) - - if keepdims: - return arr - else: - return squeeze(arr, axis) + return squeeze(arr, axis) def sum( @@ -389,31 +384,29 @@ def var( if correction < 0: raise ValueError("Correction must be non-negative in std") - axis_list = [] if axis is not None: axis_list = list(axis) if isinstance(axis, tuple) else [axis] - - resp = generic_msg( - cmd=f"stats{x.ndim}D", - args={ - "x": x._array, - "comp": "var", - "ddof": correction, - "nAxes": len(axis_list), - "axis": axis_list, - "skipNan": True, - }, + else: + axis_list = list(range(x.ndim)) + + arr = Array._new( + create_pdarray( + generic_msg( + cmd=f"varReduce<{x.dtype},{x.ndim}>", + args={ + "x": x._array, + "ddof": correction, + "axes": axis_list, + "skipNan": True, + }, + ) + ) ) - if axis is None or x.ndim == 1: - return Array._new(scalar_array(parse_single_value(resp))) + if keepdims or axis is None: + return arr else: - arr = Array._new(create_pdarray(resp)) - - if keepdims: - return arr - else: - return squeeze(arr, axis) + return squeeze(arr, axis) def _prod_sum_dtype(dtype: Dtype) -> Dtype: @@ -430,10 +423,12 @@ def _prod_sum_dtype(dtype: Dtype) -> Dtype: def cumulative_sum( - x: Array, /, *, + x: Array, + /, + *, axis: Optional[int] = None, dtype: Optional[Dtype] = None, - include_initial: bool = False + include_initial: bool = False, ) -> Array: """ Compute the cumulative sum of the elements of an array along a given axis. @@ -450,12 +445,18 @@ def cumulative_sum( include_initial : bool, optional Whether to include the initial value as the first element of the output. """ + + if dtype is None: + x_ = x + else: + x_ = akcast(x, dtype) + resp = generic_msg( - cmd=f"cumSum{x.ndim}D", + cmd=f"cumSum<{x_.dtype},{x.ndim}>", args={ - "x": x._array, + "x": x_._array, "axis": axis if axis is not None else 0, - "include_initial": include_initial, + "includeInitial": include_initial, }, ) diff --git a/arkouda/array_api/utility_functions.py b/arkouda/array_api/utility_functions.py index 2158bae9c2..69875d1ef6 100644 --- a/arkouda/array_api/utility_functions.py +++ b/arkouda/array_api/utility_functions.py @@ -125,8 +125,8 @@ def diff(a: Array, /, n: int = 1, axis: int = -1, prepend=None, append=None) -> def pad( array: Array, pad_width, # Union[int, Tuple[int, int], Tuple[Tuple[int, int], ...]] - mode='constant', - **kwargs + mode="constant", + **kwargs, ) -> Array: """ Pad an array. @@ -143,13 +143,13 @@ def pad( Padding mode. Only 'constant' is currently supported. Use the `constant_values` keyword argument to specify the padding value or values (in the same format as `pad_width`). """ - if mode != 'constant': + if mode != "constant": raise NotImplementedError(f"pad mode '{mode}' is not supported") - if 'constant_values' not in kwargs: + if "constant_values" not in kwargs: cvals = 0 else: - cvals = kwargs['constant_values'] + cvals = kwargs["constant_values"] if isinstance(pad_width, int): pad_widths_b = [pad_width] * array.ndim diff --git a/arkouda/dataframe.py b/arkouda/dataframe.py index 494de5e6d4..f5348fe844 100644 --- a/arkouda/dataframe.py +++ b/arkouda/dataframe.py @@ -4490,17 +4490,16 @@ def numeric_help(d): d = Categorical(d) return d if isinstance(d, pdarray) else d.codes - args = { - "size": len(self.columns.values), - "columns": self.columns.values, - "data_names": [numeric_help(self[c]) for c in self.columns.values], - } - - ret_dict = json.loads(generic_msg(cmd="corrMatrix", args=args)) - return DataFrame( - {c: create_pdarray(ret_dict[c]) for c in self.columns.values}, - index=array(self.columns.values), - ) + corrs = {} + for c1 in self.columns.values: + corrs[c1] = np.zeros(len(self.columns.values)) + for i, c2 in enumerate(self.columns.values): + if c1 == c2: + corrs[c1][i] = 1 + else: + corrs[c1][i] = numeric_help(self[c1]).corr(numeric_help(self[c2])) + + return DataFrame({c: array(v) for c, v in corrs.items()}, index=array(self.columns.values)) @typechecked def merge( diff --git a/arkouda/pdarrayclass.py b/arkouda/pdarrayclass.py index 060056636e..2d91c261b4 100644 --- a/arkouda/pdarrayclass.py +++ b/arkouda/pdarrayclass.py @@ -150,7 +150,7 @@ def _reshape(array: pdarray, shape: Tuple[int, ...]): """ return create_pdarray( generic_msg( - cmd=f"reshape{array.ndim}Dx{len(shape)}D", + cmd=f"reshape<{array.dtype},{array.ndim},{len(shape)}>", args={ "name": array, "shape": shape, @@ -167,7 +167,7 @@ def _squeeze(array: pdarray, degen_axes: List[int]): """ return create_pdarray( generic_msg( - cmd=f"squeeze{array.ndim}Dx{array.ndim-len(degen_axes)}D", + cmd=f"squeeze<{array.dtype},{array.ndim},{array.ndim-len(degen_axes)}>", args={ "name": array, "nAxes": len(degen_axes), @@ -2568,6 +2568,42 @@ def create_pdarray(repMsg: str, max_bits=None) -> pdarray: return pdarray(name, dtype(mydtype), size, ndim, shape, itemsize, max_bits) +@typechecked +def create_pdarrays(repMsg: str) -> List[pdarray]: + """ + Return a list of pdarray instances pointing to arrays created by the + arkouda server. + + Parameters + ---------- + repMsg : str + A JSON list of space delimited strings, each containing the pdarray + name, datatype, size, + + Returns + ------- + List[pdarray] + A list of pdarrays with the same attributes and data as the pdarrays + + Raises + ------ + ValueError + If there's an error in parsing the repMsg parameter into the six + values needed to create the pdarray instance + RuntimeError + Raised if a server-side error is thrown in the process of creating + the pdarray instance + """ + + # TODO: maybe add more robust json parsing here + try: + repMsg = repMsg.strip("[]") + responses = [r.strip().strip('\"') for r in repMsg.split("\",")] + return [create_pdarray(response) for response in responses] + except Exception as e: + raise ValueError(e) + + def clear() -> None: """ Send a clear message to clear all unregistered data from the server symbol table @@ -2912,8 +2948,8 @@ def mean(pda: pdarray) -> np.float64: """ return parse_single_value( generic_msg( - cmd=f"stats{pda.ndim}D", - args={"x": pda, "comp": "mean", "nAxes": 0, "axis": [], "ddof": 0, "skipNan": False}, + cmd=f"mean<{pda.dtype},{pda.ndim}>", + args={"x": pda, "skipNan": False}, ) ) @@ -2964,8 +3000,8 @@ def var(pda: pdarray, ddof: int_scalars = 0) -> np.float64: raise ValueError("var: ddof must be less than number of values") return parse_single_value( generic_msg( - cmd=f"stats{pda.ndim}D", - args={"x": pda, "comp": "var", "ddof": ddof, "nAxes": 0, "axis": [], "skipNan": False}, + cmd=f"var<{pda.dtype},{pda.ndim}>", + args={"x": pda, "ddof": ddof, "skipNan": False}, ) ) @@ -3020,8 +3056,8 @@ def std(pda: pdarray, ddof: int_scalars = 0) -> np.float64: raise ValueError("ddof must be an integer 0 or greater") return parse_single_value( generic_msg( - cmd=f"stats{pda.ndim}D", - args={"x": pda, "comp": "std", "ddof": ddof, "nAxes": 0, "axis": [], "skipNan": False}, + cmd=f"std<{pda.dtype},{pda.ndim}>", + args={"x": pda, "ddof": ddof, "skipNan": False}, ) ) @@ -3059,7 +3095,12 @@ def cov(x: pdarray, y: pdarray) -> np.float64: The covariance is calculated by ``cov = ((x - x.mean()) * (y - y.mean())).sum() / (x.size - 1)``. """ - return parse_single_value(generic_msg(cmd="cov", args={"x": x, "y": y})) + return parse_single_value( + generic_msg( + cmd=f"cov<{x.dtype},{x.ndim},{y.dtype},{y.ndim}>", + args={"x": x, "y": y} + ) + ) @typechecked @@ -3095,7 +3136,12 @@ def corr(x: pdarray, y: pdarray) -> np.float64: The correlation is calculated by cov(x, y) / (x.std(ddof=1) * y.std(ddof=1)) """ - return parse_single_value(generic_msg(cmd="corr", args={"x": x, "y": y})) + return parse_single_value( + generic_msg( + cmd=f"corr<{x.dtype},{x.ndim},{y.dtype},{y.ndim}>", + args={"x": x, "y": y} + ) + ) @typechecked @@ -3941,7 +3987,7 @@ def broadcast_to_shape(pda: pdarray, shape: Tuple[int, ...]) -> pdarray: cast( str, generic_msg( - cmd=f"broadcastTo{pda.ndim}Dx{len(shape)}D", + cmd=f"broadcast<{pda.dtype},{pda.ndim},{len(shape)}>", args={ "name": pda, "shape": shape, diff --git a/arkouda/pdarraycreation.py b/arkouda/pdarraycreation.py index 52179eb516..9270efb093 100644 --- a/arkouda/pdarraycreation.py +++ b/arkouda/pdarraycreation.py @@ -1,5 +1,5 @@ import itertools -from typing import Iterable, List, Optional, Tuple, Union, cast +from typing import Iterable, List, Optional, Tuple, Union, cast, Any import numpy as np import pandas as pd @@ -45,6 +45,7 @@ "random_strings_lognormal", "from_series", "bigint_from_uint_arrays", + "promote_to_common_dtype", ] @@ -304,6 +305,48 @@ def array( return create_pdarray(rep_msg) if dtype is None else akcast(create_pdarray(rep_msg), dtype) +def promote_to_common_dtype(arrays: List[pdarray]) -> Tuple[Any, List[pdarray]]: + """ + Promote a list of pdarrays to a common dtype. + + Parameters + ---------- + arrays : List[pdarray] + List of pdarrays to promote + + Returns + ------- + dtype, List[pdarray] + The common dtype of the pdarrays and the list of pdarrays promoted to that dtype + + Raises + ------ + TypeError + Raised if the pdarrays are not all of the same dtype + + See Also + -------- + pdarray.promote_dtype + + Examples + -------- + >>> a = ak.arange(5) + >>> b = ak.ones(5, dtype=ak.float64) + >>> dtype, promoted = promote_to_common_dtype([a, b]) + >>> dtype + dtype(float64) + >>> all(isinstance(p, pdarray) and p.dtype == dtype for p in promoted) + True + """ + # find the common dtype of the input arrays + dt = np.common_type(*[np.empty(0, dtype=a.dtype) for a in arrays]) + + # cast the input arrays to the output dtype if necessary + arrays = [a.astype(dt) if a.dtype != dt else a for a in arrays] + + return (akdtype(dt), arrays) + + def _array_memview(a) -> memoryview: if (get_byteorder(a.dtype) == "<" and get_server_byteorder() == "big") or ( get_byteorder(a.dtype) == ">" and get_server_byteorder() == "little" diff --git a/serverConfig.json b/registration-config.json similarity index 54% rename from serverConfig.json rename to registration-config.json index 129d352720..d4ea9e3c04 100644 --- a/serverConfig.json +++ b/registration-config.json @@ -1,5 +1,16 @@ { - "max_array_dims": 1, + "parameter_classes": { + "NOTE!!!": "run 'make register-commands' after changing any of these values", + "array": { + "nd": [1], + "dtype": [ + "int", + "uint", + "real", + "bool" + ] + } + }, "supported_scalar_types": { "uint8": true, "uint16": false, diff --git a/src/CommandMap.chpl b/src/CommandMap.chpl index 4acda0e1f9..32c2ec38aa 100644 --- a/src/CommandMap.chpl +++ b/src/CommandMap.chpl @@ -18,8 +18,8 @@ module CommandMap { } private var f = akMsgSign; - var commandMap: map(string, f.type); - var moduleMap: map(string, string); + var commandMap: map(string, f.type); // cmd-name => function + var moduleMap: map(string, (string, int)); // cmd-name => (module-name, line number) use Set; var usedModules: set(string); @@ -28,13 +28,15 @@ module CommandMap { * This binds a server command to its corresponding function matching the standard * function signature & MsgTuple return type */ - proc registerFunction(cmd: string, fcf: f.type) { - commandMap.add(cmd, fcf); - } + proc registerFunction(cmd: string, fcf: f.type, modName: string = "", line: int = -1) { + if commandMap.contains(cmd) { + writef("Warning: Command '%s' already registered. Ignoring registration from [%s:%d]", cmd, modName, line); + } else { + commandMap.add(cmd, fcf); - proc registerFunction(cmd: string, fcf: f.type, modName: string) { - commandMap.add(cmd, fcf); - moduleMap.add(cmd, modName); + if modName != "" then + moduleMap.add(cmd, (modName, line)); + } } proc writeUsedModulesJson(ref mods: set(string)) { @@ -75,7 +77,7 @@ module CommandMap { proc executeCommand(cmd: string, msgArgs, st): MsgTuple throws { var response: MsgTuple; if commandMap.contains(cmd) { - if moduleMap.contains(cmd) then usedModules.add(moduleMap[cmd]); + if moduleMap.contains(cmd) then usedModules.add(moduleMap[cmd][0]); try { response = commandMap[cmd](cmd, msgArgs, st); } catch e { diff --git a/src/ManipulationMsg.chpl b/src/ManipulationMsg.chpl index 08bb511f82..785bfce59c 100644 --- a/src/ManipulationMsg.chpl +++ b/src/ManipulationMsg.chpl @@ -58,87 +58,84 @@ module ManipulationMsg { https://data-apis.org/array-api/latest/API_specification/generated/array_api.broadcast_to.html#array_api.broadcast_to */ - @arkouda.registerNDPermInc + @arkouda.instantiateAndRegister(prefix='broadcast') proc broadcastToMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, - param ndIn: int, // rank of the array to be broadcast - param ndOut: int // rank of the result array + type array_dtype, + param array_nd_in: int, + param array_nd_out: int ): MsgTuple throws { const name = msgArgs["name"], - shapeOut = msgArgs["shape"].toScalarTuple(int, ndOut); + shapeOut = msgArgs["shape"].toScalarTuple(int, array_nd_out); - var gEnt = st[name]: borrowed GenSymEntry; + var eIn = toSymEntry(st[name]: borrowed GenSymEntry, array_dtype, array_nd_in), + eOut = createSymEntry((...shapeOut), array_dtype); - proc doBroadcast(type t): MsgTuple throws { - var eIn = toSymEntry(gEnt, t, ndIn), - eOut = createSymEntry((...shapeOut), t); + if array_nd_in == array_nd_out && eIn.tupShape == shapeOut { + // no broadcast necessary, copy the array + eOut.a = eIn.a; + } else { + // ensure that 'shapeOut' is a valid broadcast of 'eIn.tupShape' + // and determine which dimensions will require promoted assignment + var (valid, bcDims) = checkValidBroadcast(eIn.tupShape, shapeOut); - if ndIn == ndOut && eIn.tupShape == shapeOut { - // no broadcast necessary, copy the array - eOut.a = eIn.a; + if !valid { + const errorMsg = "Invalid broadcast: " + eIn.tupShape:string + " -> " + shapeOut:string; + mLogger.error(getModuleName(),getRoutineName(),getLineNumber(),errorMsg); + return new MsgTuple(errorMsg, MsgType.ERROR); } else { - // ensure that 'shapeOut' is a valid broadcast of 'eIn.tupShape' - // and determine which dimensions will require promoted assignment - var (valid, bcDims) = checkValidBroadcast(eIn.tupShape, shapeOut); - - if !valid { - const errorMsg = "Invalid broadcast: " + eIn.tupShape:string + " -> " + shapeOut:string; - mLogger.error(getModuleName(),getRoutineName(),getLineNumber(),errorMsg); - return new MsgTuple(errorMsg, MsgType.ERROR); - } else { - // use List; - // var bcDimsList = new list(int); - // for i in 0.. {0..0, 0..<4, 0..0} - // - for 'nonBCIndex' = (0, 0, 0), outSliceIdx = (0..<5, 0..0, 0..<3) - // - for 'nonBCIndex' = (0, 1, 0), outSliceIdx = (0..<5, 1..1, 0..<3) - // - etc. - // */ - // forall nonBCIndex in domOffAxis(eOut.a.domain, bcDimsList.toArray()) { - // const nbcT = if ndOut == 1 then (nonBCIndex,) else nonBCIndex; - - - // var outSliceIdx: ndOut*range; - // for i in 0.. {0..0, 0..<4, 0..0} + // - for 'nonBCIndex' = (0, 0, 0), outSliceIdx = (0..<5, 0..0, 0..<3) + // - for 'nonBCIndex' = (0, 1, 0), outSliceIdx = (0..<5, 1..1, 0..<3) + // - etc. + // */ + // forall nonBCIndex in domOffAxis(eOut.a.domain, bcDimsList.toArray()) { + // const nbcT = if ndOut == 1 then (nonBCIndex,) else nonBCIndex; + + + // var outSliceIdx: ndOut*range; + // for i in 0.. array_nd_out + { + return MsgTuple.error( + "Cannot broadcast from higher (%i) dimensional to lower (%i) dimensional array" + .format(array_nd_in, array_nd_out) + ); } proc checkValidBroadcast(from: ?Nf*int, to: ?Nt*int): (bool, Nf*bool) { @@ -158,60 +155,36 @@ module ManipulationMsg { } // https://data-apis.org/array-api/latest/API_specification/generated/array_api.concat.html#array_api.concat - @arkouda.registerND - proc concatMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, param nd: int): MsgTuple throws { + @arkouda.instantiateAndRegister(prefix='concat') + proc concatMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype, param array_nd: int): MsgTuple throws { param pn = Reflection.getRoutineName(); const nArrays = msgArgs["n"].toScalar(int), names = msgArgs["names"].toScalarArray(string, nArrays), - axis = msgArgs["axis"].getPositiveIntValue(nd); + axis = msgArgs["axis"].getPositiveIntValue(array_nd); - var gEnts = getGenericEntries(names, st); + const eIns = for n in names do st[n]: borrowed SymEntry(array_dtype, array_nd), + shapes = [i in 0.. array_nd_in { + return MsgTuple.error( + "Cannot squeeze %iD array into %iD array; output array must have fewer dimensions than input" + .format(array_nd_in, array_nd_out) + ); + } + param pn = Reflection.getRoutineName(); const name = msgArgs["name"], nAxes = msgArgs["nAxes"].toScalar(int), axes = msgArgs["axes"].toScalarArray(int, nAxes); - var gEnt = st[name]: borrowed GenSymEntry; + var eIn = st[name]: borrowed SymEntry(array_dtype, array_nd_in), + (valid, shape, mapping) = validateSqueeze(eIn.tupShape, axes, array_nd_out); - proc doSqueeze(type t): MsgTuple throws { - const eIn = toSymEntry(gEnt, t, ndIn), - (valid, shape, mapping) = validateSqueeze(eIn.tupShape, axes, ndOut); - - if !valid { - const errMsg = "Unable to squeeze array with shape %? along axes %? into a %iD array".format(eIn.tupShape, axes, ndOut); - mLogger.error(getModuleName(),pn,getLineNumber(),errMsg); - return new MsgTuple(errMsg,MsgType.ERROR); - } else { - var eOut = createSymEntry((...shape), t); - - // copy the data from the input array to the output array - forall idx in eOut.a.domain with ( - var agg = newSrcAggregator(t), - const imap = new squeezeIndexMapper(ndIn, ndOut, mapping) - ) do - agg.copy(eOut.a[idx], eIn.a[imap(if ndOut==1 then (idx,) else idx)]); + if !valid { + const errMsg = "Unable to squeeze array with shape %? along axes %? into a %iD array".format(eIn.tupShape, axes, array_nd_out); + mLogger.error(getModuleName(),pn,getLineNumber(),errMsg); + return new MsgTuple(errMsg,MsgType.ERROR); + } else { + var eOut = createSymEntry((...shape), array_dtype); - return st.insert(eOut); - } - } + // copy the data from the input array to the output array + forall idx in eOut.a.domain with ( + var agg = newSrcAggregator(array_dtype), + const imap = new squeezeIndexMapper(array_nd_in, array_nd_out, mapping) + ) do + agg.copy(eOut.a[idx], eIn.a[imap(if array_nd_out==1 then (idx,) else idx)]); - select gEnt.dtype { - when DType.Int64 do return doSqueeze(int); - when DType.UInt64 do return doSqueeze(uint); - when DType.Float64 do return doSqueeze(real); - when DType.Bool do return doSqueeze(bool); - otherwise { - var errorMsg = notImplementedError(pn,dtype2str(gEnt.dtype)); - mLogger.error(getModuleName(),pn,getLineNumber(),errorMsg); - return new MsgTuple(errorMsg,MsgType.ERROR); - } + return st.insert(eOut); } } @@ -856,12 +683,12 @@ module ManipulationMsg { } // https://data-apis.org/array-api/latest/API_specification/generated/array_api.stack.html#array_api.stack - @arkouda.registerND - proc stackMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, param nd: int): MsgTuple throws { + @arkouda.instantiateAndRegister(prefix='stack') + proc stackMsg(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype, param array_nd: int): MsgTuple throws { param pn = Reflection.getRoutineName(); - if nd == MaxArrayDims { - const errMsg = "Cannot stack arrays with rank %i, as this would result an an array with rank %i".format(nd, nd+1) + + if array_nd == MaxArrayDims { + const errMsg = "Cannot stack arrays with rank %i, as this would result an an array with rank %i".format(array_nd, array_nd+1) + ", exceeding the server's configured maximum of %i. ".format(MaxArrayDims) + "Please update the configuration and recompile to support higher-dimensional arrays."; mLogger.error(getModuleName(),pn,getLineNumber(),errMsg); @@ -870,50 +697,31 @@ module ManipulationMsg { const nArrays = msgArgs["n"].toScalar(int), names = msgArgs["names"].toScalarArray(string, nArrays), - axis = msgArgs["axis"].getPositiveIntValue(nd+1); - - var gEnts = getGenericEntries(names, st); - - // confirm that all arrays have the same dtype and shape - // (type promotion needs to be completed before calling 'stack') - const dt = gEnts[0]!.dtype, - sh = gEnts[0]!.shape; - for i in 1.. 1 - { - const eIn = toSymEntry(gArr, t, nd), - eObj = toSymEntry(gObj, int, 1); - - var shapeOut: nd*int; - for i in 0.. 1 - { - const eIn = toSymEntry(gEnt, t, nd); + return (+ reduce ((x:real - mx) * (y:real - my))) / (dx.size - 1):real; + } - var outShape = eIn.tupShape; - if includeInitial then outShape[axis] += 1; - var eOut = st.addEntry(rname, outShape, t); + // above registration will instantiate `cov` for all combinations of array ranks + // even though it is only valid when the ranks are the same + // (respecting the where clause in the signature is future work for 'registerCommand') + proc cov(const ref x: [?dx], const ref y: [?dy]): real throws + where dx.rank != dy.rank + { + throw new Error("x and y must have the same rank"); + } - const DD = domOffAxis(eIn.a.domain, axis); - for idx in DD { - // TODO: avoid making a copy of the slice here - const sliceDom = domOnAxis(eIn.a.domain, idx, axis), - slice = removeDegenRanks(eIn.a[sliceDom], 1); - const cs = + scan slice; + @arkouda.registerCommand() + proc corr(const ref x: [?dx] ?tx, const ref y: [?dy] ?ty): real throws + where dx.rank == dy.rank + { + if dx.shape != dy.shape then + throw new Error("x and y must have the same shape"); - if includeInitial { - // put the additive identity in the first element - eOut.a[idx] = 0:t; + const mx = mean(x), + my = mean(y); - forall i in sliceDom { - var iShift = i; - iShift[axis] += 1; - eOut.a[iShift] = cs[i[axis]]; - } - } else { - forall i in sliceDom do eOut.a[i] = cs[i[axis]]; - } - } + return cov(x, y) / (std(x, 1) * std(y, 1)); + } - const repMsg = "created " + st.attrib(rname); - sLogger.info(getModuleName(),pn,getLineNumber(),repMsg); - return new MsgTuple(repMsg, MsgType.NORMAL); - } + // above registration will instantiate `corr` for all combinations of array ranks + // even though it is only valid when the ranks are the same + // (respecting the where clause in the signature is future work for 'registerCommand') + proc corr(const ref x: [?dx], const ref y: [?dy]): real throws + where dx.rank != dy.rank + { + throw new Error("x and y must have the same rank"); + } - select gEnt.dtype { - when DType.Int64 do return computeCumSum(int); - when DType.UInt64 do return computeCumSum(uint); - when DType.Float64 do return computeCumSum(real); - otherwise { - const errorMsg = notImplementedError(pn,dtype2str(gEnt.dtype)); - sLogger.error(getModuleName(),pn,getLineNumber(),errorMsg); - return new MsgTuple(errorMsg,MsgType.ERROR); - } + @arkouda.registerCommand() + proc cumSum(const ref x: [?d] ?t, axis: int, includeInitial: bool): [] t throws { + if d.rank == 1 { + var cs = makeDistArray(if includeInitial then x.size+1 else x.size, t); + cs[if includeInitial then 1.. else 0..] = (+ scan x):t; + return cs; + } else { + var cs = makeDistArray(if includeInitial then expandedDomain(d, axis) else d, t); + + forall (slice, _) in axisSlices(d, new list([axis])) { + const xSlice = removeDegenRanks(x[slice], 1), + csSlice = (+ scan xSlice):t; + + for idx in slice { + var csIdx = idx; + if includeInitial then csIdx[axis] += 1; + cs[csIdx] = csSlice[idx[axis]]; + } } + return cs; + } } - use CommandMap; - registerFunction("cov", covMsg, getModuleName()); - registerFunction("corr", corrMsg, getModuleName()); - registerFunction("corrMatrix", corrMatrixMsg, getModuleName()); + private proc expandedDomain(d: domain(?), axis: int): domain(?) { + var rngs: d.rank*range; + for param i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + return domOnAxis(D, idx, new list(axes)); + } + + proc domOnAxis(D: domain(?), idx: D.rank*int, const ref axes: list(int)): domain(?) throws { + if axes.size > D.rank then + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..= D.rank then - throw new Error("Cannot create a " + axes.size:string + "-dimensional slice from a " + D.rank:string + "-dimensional domain"); + return domOffAxis(D, new list(axes)); + } + + proc domOffAxis(D: domain(?), const ref axes: list(int)): domain(?) throws { + if axes.size > D.rank then + throw new Error("Cannot create a %i dimensional slice from a %i dimensional domain".format(axes.size, D.rank)); var outDims: D.rank*range; - label ranks for i in 0..', ark_broadcast_int_1_1, 'ManipulationMsg', 62); + +proc ark_broadcast_uint_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.broadcastToMsg(cmd, msgArgs, st, array_dtype=uint, array_nd_in=1, array_nd_out=1); +registerFunction('broadcast', ark_broadcast_uint_1_1, 'ManipulationMsg', 62); + +proc ark_broadcast_real_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.broadcastToMsg(cmd, msgArgs, st, array_dtype=real, array_nd_in=1, array_nd_out=1); +registerFunction('broadcast', ark_broadcast_real_1_1, 'ManipulationMsg', 62); + +proc ark_broadcast_bool_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.broadcastToMsg(cmd, msgArgs, st, array_dtype=bool, array_nd_in=1, array_nd_out=1); +registerFunction('broadcast', ark_broadcast_bool_1_1, 'ManipulationMsg', 62); + +proc ark_concat_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('concat', ark_concat_int_1, 'ManipulationMsg', 159); + +proc ark_concat_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('concat', ark_concat_uint_1, 'ManipulationMsg', 159); + +proc ark_concat_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('concat', ark_concat_real_1, 'ManipulationMsg', 159); + +proc ark_concat_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('concat', ark_concat_bool_1, 'ManipulationMsg', 159); + +proc ark_concatFlat_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatFlatMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('concatFlat', ark_concatFlat_int_1, 'ManipulationMsg', 215); + +proc ark_concatFlat_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatFlatMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('concatFlat', ark_concatFlat_uint_1, 'ManipulationMsg', 215); + +proc ark_concatFlat_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatFlatMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('concatFlat', ark_concatFlat_real_1, 'ManipulationMsg', 215); + +proc ark_concatFlat_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.concatFlatMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('concatFlat', ark_concatFlat_bool_1, 'ManipulationMsg', 215); + +proc ark_expandDims_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.expandDimsMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('expandDims', ark_expandDims_int_1, 'ManipulationMsg', 239); + +proc ark_expandDims_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.expandDimsMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('expandDims', ark_expandDims_uint_1, 'ManipulationMsg', 239); + +proc ark_expandDims_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.expandDimsMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('expandDims', ark_expandDims_real_1, 'ManipulationMsg', 239); + +proc ark_expandDims_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.expandDimsMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('expandDims', ark_expandDims_bool_1, 'ManipulationMsg', 239); + +proc ark_flip_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('flip', ark_flip_int_1, 'ManipulationMsg', 291); + +proc ark_flip_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('flip', ark_flip_uint_1, 'ManipulationMsg', 291); + +proc ark_flip_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('flip', ark_flip_real_1, 'ManipulationMsg', 291); + +proc ark_flip_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('flip', ark_flip_bool_1, 'ManipulationMsg', 291); + +proc ark_flipAll_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipAllMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('flipAll', ark_flipAll_int_1, 'ManipulationMsg', 359); + +proc ark_flipAll_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipAllMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('flipAll', ark_flipAll_uint_1, 'ManipulationMsg', 359); + +proc ark_flipAll_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipAllMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('flipAll', ark_flipAll_real_1, 'ManipulationMsg', 359); + +proc ark_flipAll_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.flipAllMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('flipAll', ark_flipAll_bool_1, 'ManipulationMsg', 359); + +proc ark_permuteDims_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.permuteDims(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('permuteDims', ark_permuteDims_int_1, 'ManipulationMsg', 390); + +proc ark_permuteDims_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.permuteDims(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('permuteDims', ark_permuteDims_uint_1, 'ManipulationMsg', 390); + +proc ark_permuteDims_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.permuteDims(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('permuteDims', ark_permuteDims_real_1, 'ManipulationMsg', 390); + +proc ark_permuteDims_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.permuteDims(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('permuteDims', ark_permuteDims_bool_1, 'ManipulationMsg', 390); + +proc ark_reshape_int_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.reshapeMsg(cmd, msgArgs, st, array_dtype=int, array_nd_in=1, array_nd_out=1); +registerFunction('reshape', ark_reshape_int_1_1, 'ManipulationMsg', 440); + +proc ark_reshape_uint_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.reshapeMsg(cmd, msgArgs, st, array_dtype=uint, array_nd_in=1, array_nd_out=1); +registerFunction('reshape', ark_reshape_uint_1_1, 'ManipulationMsg', 440); + +proc ark_reshape_real_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.reshapeMsg(cmd, msgArgs, st, array_dtype=real, array_nd_in=1, array_nd_out=1); +registerFunction('reshape', ark_reshape_real_1_1, 'ManipulationMsg', 440); + +proc ark_reshape_bool_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.reshapeMsg(cmd, msgArgs, st, array_dtype=bool, array_nd_in=1, array_nd_out=1); +registerFunction('reshape', ark_reshape_bool_1_1, 'ManipulationMsg', 440); + +proc ark_roll_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('roll', ark_roll_int_1, 'ManipulationMsg', 522); + +proc ark_roll_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('roll', ark_roll_uint_1, 'ManipulationMsg', 522); + +proc ark_roll_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('roll', ark_roll_real_1, 'ManipulationMsg', 522); + +proc ark_roll_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('roll', ark_roll_bool_1, 'ManipulationMsg', 522); + +proc ark_rollFlattened_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollFlattenedMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('rollFlattened', ark_rollFlattened_int_1, 'ManipulationMsg', 587); + +proc ark_rollFlattened_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollFlattenedMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('rollFlattened', ark_rollFlattened_uint_1, 'ManipulationMsg', 587); + +proc ark_rollFlattened_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollFlattenedMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('rollFlattened', ark_rollFlattened_real_1, 'ManipulationMsg', 587); + +proc ark_rollFlattened_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.rollFlattenedMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('rollFlattened', ark_rollFlattened_bool_1, 'ManipulationMsg', 587); + +proc ark_squeeze_int_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.squeezeMsg(cmd, msgArgs, st, array_dtype=int, array_nd_in=1, array_nd_out=1); +registerFunction('squeeze', ark_squeeze_int_1_1, 'ManipulationMsg', 608); + +proc ark_squeeze_uint_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.squeezeMsg(cmd, msgArgs, st, array_dtype=uint, array_nd_in=1, array_nd_out=1); +registerFunction('squeeze', ark_squeeze_uint_1_1, 'ManipulationMsg', 608); + +proc ark_squeeze_real_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.squeezeMsg(cmd, msgArgs, st, array_dtype=real, array_nd_in=1, array_nd_out=1); +registerFunction('squeeze', ark_squeeze_real_1_1, 'ManipulationMsg', 608); + +proc ark_squeeze_bool_1_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.squeezeMsg(cmd, msgArgs, st, array_dtype=bool, array_nd_in=1, array_nd_out=1); +registerFunction('squeeze', ark_squeeze_bool_1_1, 'ManipulationMsg', 608); + +proc ark_stack_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.stackMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('stack', ark_stack_int_1, 'ManipulationMsg', 687); + +proc ark_stack_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.stackMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('stack', ark_stack_uint_1, 'ManipulationMsg', 687); + +proc ark_stack_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.stackMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('stack', ark_stack_real_1, 'ManipulationMsg', 687); + +proc ark_stack_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.stackMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('stack', ark_stack_bool_1, 'ManipulationMsg', 687); + +proc ark_tile_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.tileMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('tile', ark_tile_int_1, 'ManipulationMsg', 778); + +proc ark_tile_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.tileMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('tile', ark_tile_uint_1, 'ManipulationMsg', 778); + +proc ark_tile_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.tileMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('tile', ark_tile_real_1, 'ManipulationMsg', 778); + +proc ark_tile_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.tileMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('tile', ark_tile_bool_1, 'ManipulationMsg', 778); + +proc ark_unstack_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.unstackMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('unstack', ark_unstack_int_1, 'ManipulationMsg', 819); + +proc ark_unstack_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.unstackMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('unstack', ark_unstack_uint_1, 'ManipulationMsg', 819); + +proc ark_unstack_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.unstackMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('unstack', ark_unstack_real_1, 'ManipulationMsg', 819); + +proc ark_unstack_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.unstackMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('unstack', ark_unstack_bool_1, 'ManipulationMsg', 819); + +proc ark_repeatFlat_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.repeatFlatMsg(cmd, msgArgs, st, array_dtype=int, array_nd=1); +registerFunction('repeatFlat', ark_repeatFlat_int_1, 'ManipulationMsg', 903); + +proc ark_repeatFlat_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.repeatFlatMsg(cmd, msgArgs, st, array_dtype=uint, array_nd=1); +registerFunction('repeatFlat', ark_repeatFlat_uint_1, 'ManipulationMsg', 903); + +proc ark_repeatFlat_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.repeatFlatMsg(cmd, msgArgs, st, array_dtype=real, array_nd=1); +registerFunction('repeatFlat', ark_repeatFlat_real_1, 'ManipulationMsg', 903); + +proc ark_repeatFlat_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ManipulationMsg.repeatFlatMsg(cmd, msgArgs, st, array_dtype=bool, array_nd=1); +registerFunction('repeatFlat', ark_repeatFlat_bool_1, 'ManipulationMsg', 903); + +import OperatorMsg; + +import ParquetMsg; + +import RandMsg; + +import ReductionMsg; + +import RegistrationMsg; + +import SegmentedMsg; + +import SequenceMsg; + +import SortMsg; + +import StatsMsg; + +proc ark_reg_mean_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var skipNan = msgArgs['skipNan'].toScalar(bool); + var ark_result = StatsMsg.mean(x,skipNan); + + return MsgTuple.fromScalar(ark_result); +} + +proc ark_mean_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_mean_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('mean', ark_mean_int_1, 'StatsMsg', 25); + +proc ark_mean_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_mean_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('mean', ark_mean_uint_1, 'StatsMsg', 25); + +proc ark_mean_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_mean_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('mean', ark_mean_real_1, 'StatsMsg', 25); + +proc ark_mean_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_mean_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('mean', ark_mean_bool_1, 'StatsMsg', 25); + +proc ark_reg_meanReduce_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var skipNan = msgArgs['skipNan'].toScalar(bool); + var axes = msgArgs['axes'].toScalarList(int); + var ark_result = StatsMsg.meanReduce(x,skipNan,axes); + var ark_result_symbol = new shared SymEntry(ark_result); + + return st.insert(ark_result_symbol); +} + +proc ark_meanReduce_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_meanReduce_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('meanReduce', ark_meanReduce_int_1, 'StatsMsg', 32); + +proc ark_meanReduce_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_meanReduce_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('meanReduce', ark_meanReduce_uint_1, 'StatsMsg', 32); + +proc ark_meanReduce_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_meanReduce_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('meanReduce', ark_meanReduce_real_1, 'StatsMsg', 32); + +proc ark_meanReduce_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_meanReduce_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('meanReduce', ark_meanReduce_bool_1, 'StatsMsg', 32); + +proc ark_reg_variance_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var skipNan = msgArgs['skipNan'].toScalar(bool); + var ddof = msgArgs['ddof'].toScalar(real); + var ark_result = StatsMsg.variance(x,skipNan,ddof); + + return MsgTuple.fromScalar(ark_result); +} + +proc ark_var_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_variance_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('var', ark_var_int_1, 'StatsMsg', 43); + +proc ark_var_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_variance_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('var', ark_var_uint_1, 'StatsMsg', 43); + +proc ark_var_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_variance_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('var', ark_var_real_1, 'StatsMsg', 43); + +proc ark_var_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_variance_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('var', ark_var_bool_1, 'StatsMsg', 43); + +proc ark_reg_varReduce_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var skipNan = msgArgs['skipNan'].toScalar(bool); + var ddof = msgArgs['ddof'].toScalar(real); + var axes = msgArgs['axes'].toScalarList(int); + var ark_result = StatsMsg.varReduce(x,skipNan,ddof,axes); + var ark_result_symbol = new shared SymEntry(ark_result); + + return st.insert(ark_result_symbol); +} + +proc ark_varReduce_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_varReduce_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('varReduce', ark_varReduce_int_1, 'StatsMsg', 50); + +proc ark_varReduce_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_varReduce_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('varReduce', ark_varReduce_uint_1, 'StatsMsg', 50); + +proc ark_varReduce_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_varReduce_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('varReduce', ark_varReduce_real_1, 'StatsMsg', 50); + +proc ark_varReduce_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_varReduce_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('varReduce', ark_varReduce_bool_1, 'StatsMsg', 50); + +proc ark_reg_std_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var skipNan = msgArgs['skipNan'].toScalar(bool); + var ddof = msgArgs['ddof'].toScalar(real); + var ark_result = StatsMsg.std(x,skipNan,ddof); + + return MsgTuple.fromScalar(ark_result); +} + +proc ark_std_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_std_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('std', ark_std_int_1, 'StatsMsg', 61); + +proc ark_std_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_std_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('std', ark_std_uint_1, 'StatsMsg', 61); + +proc ark_std_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_std_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('std', ark_std_real_1, 'StatsMsg', 61); + +proc ark_std_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_std_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('std', ark_std_bool_1, 'StatsMsg', 61); + +proc ark_reg_stdReduce_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var skipNan = msgArgs['skipNan'].toScalar(bool); + var ddof = msgArgs['ddof'].toScalar(real); + var axes = msgArgs['axes'].toScalarList(int); + var ark_result = StatsMsg.stdReduce(x,skipNan,ddof,axes); + var ark_result_symbol = new shared SymEntry(ark_result); + + return st.insert(ark_result_symbol); +} + +proc ark_stdReduce_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_stdReduce_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('stdReduce', ark_stdReduce_int_1, 'StatsMsg', 68); + +proc ark_stdReduce_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_stdReduce_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('stdReduce', ark_stdReduce_uint_1, 'StatsMsg', 68); + +proc ark_stdReduce_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_stdReduce_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('stdReduce', ark_stdReduce_real_1, 'StatsMsg', 68); + +proc ark_stdReduce_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_stdReduce_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('stdReduce', ark_stdReduce_bool_1, 'StatsMsg', 68); + +proc ark_reg_cov_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int, type array_dtype_1, param array_nd_1: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var y_array_sym = st[msgArgs['y']]: SymEntry(array_dtype_1, array_nd_1); + ref y = y_array_sym.a; + var ark_result = StatsMsg.cov(x,y); + + return MsgTuple.fromScalar(ark_result); +} + +proc ark_cov_int_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('cov', ark_cov_int_1_int_1, 'StatsMsg', 79); + +proc ark_cov_int_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('cov', ark_cov_int_1_uint_1, 'StatsMsg', 79); + +proc ark_cov_int_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('cov', ark_cov_int_1_real_1, 'StatsMsg', 79); + +proc ark_cov_int_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('cov', ark_cov_int_1_bool_1, 'StatsMsg', 79); + +proc ark_cov_uint_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('cov', ark_cov_uint_1_int_1, 'StatsMsg', 79); + +proc ark_cov_uint_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('cov', ark_cov_uint_1_uint_1, 'StatsMsg', 79); + +proc ark_cov_uint_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('cov', ark_cov_uint_1_real_1, 'StatsMsg', 79); + +proc ark_cov_uint_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('cov', ark_cov_uint_1_bool_1, 'StatsMsg', 79); + +proc ark_cov_real_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('cov', ark_cov_real_1_int_1, 'StatsMsg', 79); + +proc ark_cov_real_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('cov', ark_cov_real_1_uint_1, 'StatsMsg', 79); + +proc ark_cov_real_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('cov', ark_cov_real_1_real_1, 'StatsMsg', 79); + +proc ark_cov_real_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('cov', ark_cov_real_1_bool_1, 'StatsMsg', 79); + +proc ark_cov_bool_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('cov', ark_cov_bool_1_int_1, 'StatsMsg', 79); + +proc ark_cov_bool_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('cov', ark_cov_bool_1_uint_1, 'StatsMsg', 79); + +proc ark_cov_bool_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('cov', ark_cov_bool_1_real_1, 'StatsMsg', 79); + +proc ark_cov_bool_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cov_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('cov', ark_cov_bool_1_bool_1, 'StatsMsg', 79); + +proc ark_reg_corr_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int, type array_dtype_1, param array_nd_1: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var y_array_sym = st[msgArgs['y']]: SymEntry(array_dtype_1, array_nd_1); + ref y = y_array_sym.a; + var ark_result = StatsMsg.corr(x,y); + + return MsgTuple.fromScalar(ark_result); +} + +proc ark_corr_int_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('corr', ark_corr_int_1_int_1, 'StatsMsg', 101); + +proc ark_corr_int_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('corr', ark_corr_int_1_uint_1, 'StatsMsg', 101); + +proc ark_corr_int_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('corr', ark_corr_int_1_real_1, 'StatsMsg', 101); + +proc ark_corr_int_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('corr', ark_corr_int_1_bool_1, 'StatsMsg', 101); + +proc ark_corr_uint_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('corr', ark_corr_uint_1_int_1, 'StatsMsg', 101); + +proc ark_corr_uint_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('corr', ark_corr_uint_1_uint_1, 'StatsMsg', 101); + +proc ark_corr_uint_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('corr', ark_corr_uint_1_real_1, 'StatsMsg', 101); + +proc ark_corr_uint_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('corr', ark_corr_uint_1_bool_1, 'StatsMsg', 101); + +proc ark_corr_real_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('corr', ark_corr_real_1_int_1, 'StatsMsg', 101); + +proc ark_corr_real_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('corr', ark_corr_real_1_uint_1, 'StatsMsg', 101); + +proc ark_corr_real_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('corr', ark_corr_real_1_real_1, 'StatsMsg', 101); + +proc ark_corr_real_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('corr', ark_corr_real_1_bool_1, 'StatsMsg', 101); + +proc ark_corr_bool_1_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=int, array_nd_1=1); +registerFunction('corr', ark_corr_bool_1_int_1, 'StatsMsg', 101); + +proc ark_corr_bool_1_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=uint, array_nd_1=1); +registerFunction('corr', ark_corr_bool_1_uint_1, 'StatsMsg', 101); + +proc ark_corr_bool_1_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=real, array_nd_1=1); +registerFunction('corr', ark_corr_bool_1_real_1, 'StatsMsg', 101); + +proc ark_corr_bool_1_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_corr_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1, array_dtype_1=bool, array_nd_1=1); +registerFunction('corr', ark_corr_bool_1_bool_1, 'StatsMsg', 101); + +proc ark_reg_cumSum_generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, type array_dtype_0, param array_nd_0: int): MsgTuple throws { + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + var axis = msgArgs['axis'].toScalar(int); + var includeInitial = msgArgs['includeInitial'].toScalar(bool); + var ark_result = StatsMsg.cumSum(x,axis,includeInitial); + var ark_result_symbol = new shared SymEntry(ark_result); + + return st.insert(ark_result_symbol); +} + +proc ark_cumSum_int_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cumSum_generic(cmd, msgArgs, st, array_dtype_0=int, array_nd_0=1); +registerFunction('cumSum', ark_cumSum_int_1, 'StatsMsg', 123); + +proc ark_cumSum_uint_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cumSum_generic(cmd, msgArgs, st, array_dtype_0=uint, array_nd_0=1); +registerFunction('cumSum', ark_cumSum_uint_1, 'StatsMsg', 123); + +proc ark_cumSum_real_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cumSum_generic(cmd, msgArgs, st, array_dtype_0=real, array_nd_0=1); +registerFunction('cumSum', ark_cumSum_real_1, 'StatsMsg', 123); + +proc ark_cumSum_bool_1(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws do + return ark_reg_cumSum_generic(cmd, msgArgs, st, array_dtype_0=bool, array_nd_0=1); +registerFunction('cumSum', ark_cumSum_bool_1, 'StatsMsg', 123); + +import TimeClassMsg; + +import TransferMsg; + +import UniqueMsg; + +} \ No newline at end of file diff --git a/src/registry/register_commands.py b/src/registry/register_commands.py new file mode 100644 index 0000000000..938bc23593 --- /dev/null +++ b/src/registry/register_commands.py @@ -0,0 +1,733 @@ +import chapel +import sys +import json + +registerAttr = ("arkouda.registerCommand", ["name"]) +instAndRegisterAttr = ("arkouda.instantiateAndRegister", ["prefix"]) + +# chapel types and their numpy equivalents +chapel_scalar_types = { + "string": "str", + "bool": "bool", + "int(8)": "int8", + "int(16)": "int16", + "int(32)": "int32", + "int(64)": "int64", + "int": "int64", + "uint(8)": "uint8", + "uint(16)": "uint16", + "uint(32)": "uint32", + "uint(64)": "uint64", + "uint": "uint64", + "real(32)": "float32", + "real(64)": "float64", + "real": "float64", + "complex(64)": "complex64", + "complex(128)": "complex128", + "complex": "complex128", +} + +# type and variable names from arkouda infrastructure that could conceivable be changed in the future: +ARGS_FORMAL_INTENT = "" +ARGS_FORMAL_NAME = "msgArgs" +ARGS_FORMAL_TYPE = "borrowed MessageArgs" + +SYMTAB_FORMAL_INTENT = "" +SYMTAB_FORMAL_NAME = "st" +SYMTAB_FORMAL_TYPE = "borrowed SymTab" + +ARRAY_ENTRY_CLASS_NAME = "SymEntry" + +RESPONSE_TYPE_NAME = "MsgTuple" + + +def error_message(message, details, loc=None): + if loc: + info = str(loc).split(":") + print(" [", info[0], ":", info[1], "] ", file=sys.stderr, end="") + + print("Error ", message, ": ", details, file=sys.stderr) + + +def is_registerable(fn): + """ + Check if a function is registerable as an Arkouda command + """ + if fn.is_method(): + error_message( + f"registering {fn.name()}", + "cannot register methods as commands", + fn.location(), + ) + return False + return True + + +def annotated_procs(root, attr): + """ + Find all procs in root annotated with a particular attribute + """ + for fn, _ in chapel.each_matching(root, chapel.Function): + if ag := fn.attribute_group(): + for a in ag: + if attr_call := chapel.parse_attribute(a, attr): + if is_registerable(fn): + yield fn, attr_call + + +def get_formals(fn): + """ + Get a function's formal parameters, separating them into concrete and + generic (param and type) formals + The (name, storage kind, and type expression) of each formal is returned + """ + + def info_tuple(formal): + if te := formal.type_expression(): + if isinstance(te, chapel.BracketLoop): + if te.is_maybe_array_type(): + ten = "" + else: + raise ValueError("invalid formal type expression") + elif isinstance(te, chapel.FnCall): + if ce := te.called_expression(): + if isinstance(ce, chapel.Identifier): + call_name = ce.name() + if call_name == "list": + list_type = list(te.actuals())[0].name() + if list_type not in chapel_scalar_types: + error_message( + f"registering '{fn.name()}'", + f"unsupported formal type for registration {list_type}", + ) + ten = None + else: + ten = f"list,{list_type}" + elif call_name == "borrowed": + ten = call_name + " " + list(te.actuals())[0].name() + else: + error_message( + f"registering '{fn.name()}'", + f"unsupported composite type expression for registration {call_name}", + ) + ten = None + else: + ten = None + else: + ten = None + else: + ten = te.name() + else: + ten = None + return (formal.name(), formal.storage_kind(), ten) + + con_formals = [] + gen_formals = [] + for formal in fn.formals(): + if isinstance(formal, (chapel.TupleDecl, chapel.VarArgFormal)): + raise ValueError( + "registration of procedures with tuple or vararg formals is not yet supported." + ) + elif isinstance(formal, (chapel.Formal, chapel.AnonFormal)): + if formal.storage_kind() in ["type", "param"]: + gen_formals.append(info_tuple(formal)) + else: + con_formals.append(info_tuple(formal)) + return con_formals, gen_formals + + +def stamp_generic_command(generic_proc_name, prefix, module_name, formals, line_num, is_user_proc): + """ + Create code to stamp out and register a generic command using a generic + procedure, and a set values for its generic formals. + + formals should be in the format {p1: v1, p2: v2, ...} + + the stamped command will have a name in the format: prefix + """ + + command_name = ( + prefix + + "<" + + ",".join( + [ + # if the generic formal is a 'type' convert it to its numpy dtype name + (chapel_scalar_types[v] if v in chapel_scalar_types else str(v)) + for _, v in formals.items() + ] + ) + + ">" + ) + + stamp_name = f"ark_{prefix}_" + "_".join( + [str(v).replace("(", "").replace(")", "") for _, v in formals.items()] + ) + + stamp_formal_args = ", ".join([f"{k}={v}" for k, v in formals.items()]) + + # use qualified naming if generic_proc belongs in a use defined module to avoid name conflicts + call = f"{module_name}.{generic_proc_name}" if is_user_proc else generic_proc_name + + proc = ( + f"proc {stamp_name}(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): {RESPONSE_TYPE_NAME} throws do\n" + + f" return {call}(cmd, msgArgs, st, {stamp_formal_args});\n" + + f"registerFunction('{command_name}', {stamp_name}, '{module_name}', {line_num});" + ) + return proc + + +def permutations(to_permute): + """ + Take an object in the format {p1: [v1, v2, ...], p2: [w1, w2, ...], ...} + + and return a list of permutations of all the values of the parameters in + the format: + + [{p1: v1, p2: w1, ...}, {p1: v1, p2: w2, ...}, ...] + """ + perms = [{}] + for parameter, values in to_permute.items(): + new_perms = [] + for p in perms: + for v in values: + x = p.copy() + x.update({parameter: v}) + new_perms.append(x) + perms = new_perms + return perms + + +# TODO: validate the 'parameter_classes' fields at the beginning of this script +# s.t. invalid values don't emit an error message associated with a specific proc +# (i.e., error instantiating 'meanMsg' - invalid parameter value type (list) in list '[["a", "b"], "c"]') +def parse_param_class_value(value): + """ + Parse a value from the 'parameter_classes' field in the configuration file + + Allows scalars, lists of scalars, or strings that can be evaluated as lists + """ + if isinstance(value, list): + for v in value: + if not isinstance(v, (int, float, str)): + raise ValueError( + f"Invalid parameter value type ({type(v)}) in list '{value}'" + ) + return value + elif isinstance(value, int): + return [ + value, + ] + elif isinstance(value, float): + return [ + value, + ] + elif isinstance(value, str): + # evaluate string as python code, resulting in a list + vals = eval(value) + if isinstance(vals, list): + return vals + else: + raise ValueError( + f"Could not create a list of parameter values from '{value}'" + ) + else: + raise ValueError(f"Invalid parameter value type ({type(value)}) for '{value}'") + + +def generic_permutations(config, gen_formals): + """ + Create a list of all possible permutations of the generic formals, using + values from the 'parameter_classes' field in the configuration file + """ + to_permute = {} + # TODO: check that the type annotations on param formals are scalars and that + # the values in the config file can be used as literals of that type + for formal_name, storage_kind, _ in gen_formals: + name_components = formal_name.split("_") + + if len(name_components) < 2: + raise ValueError( + f"invalid generic formal '{formal_name}; " + + "generic formals must be in the format '_[_...]' " + + "to be instantiated with values from the configuration file" + ) + + pclass = name_components[0] # ex: 'array' + pname = name_components[1] # ex: 'nd' + + if pclass not in config["parameter_classes"].keys(): + raise ValueError( + f"generic formal '{formal_name}' is not a valid parameter class; " + + "please check the 'parameter_classes' field in the configuration file" + ) + + if pname not in config["parameter_classes"][pclass].keys(): + raise ValueError( + f"parameter class '{pclass}' has no parameter '{pname}'; " + + "please check the 'parameter_classes' field in the configuration file" + ) + + to_permute[formal_name] = parse_param_class_value( + config["parameter_classes"][pclass][pname] + ) + + return permutations(to_permute) + + +def valid_generic_command_signature(fn, con_formals): + """ + Ensure that a proc's signature matches the format: + 'proc (cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws' + s.t. it's instantiations can be registered in the command table + """ + if len(con_formals) == 3: + if ( + con_formals[0][0] != "cmd" + or con_formals[0][2] != "string" + or con_formals[1][0] != ARGS_FORMAL_NAME + or con_formals[1][1] != ARGS_FORMAL_INTENT + or con_formals[1][2] != ARGS_FORMAL_TYPE + or con_formals[2][0] != SYMTAB_FORMAL_NAME + or con_formals[2][1] != SYMTAB_FORMAL_INTENT + or con_formals[2][2] != SYMTAB_FORMAL_TYPE + ): + return False + else: + return False + + if rt := fn.return_type(): + if rt.name() != RESPONSE_TYPE_NAME: + return False + else: + return False + + if not fn.throws(): + return False + + return True + + +# TODO: use var/const depending on user proc's formal intent +def unpack_array_arg(arg_name, array_count): + """ + Generate the code to unpack an array symbol from the symbol table + + 'array_count' is used to generate unique names for the dtype and nd arguments when + a procedure has multiple array-symbol formals + + Example: + ``` + var x_array_sym = st[msgArgs['x']]: SymEntry(array_dtype_0, array_nd_0); + ref x = x_array_sym.a; + ``` + + Returns the chapel code, and the specifications for the + 'etype' and 'dimensions' type-constructor arguments + """ + dtype_arg_name = "array_dtype_" + str(array_count) + nd_arg_name = "array_nd_" + str(array_count) + return ( + f"\tvar {arg_name}_array_sym = {SYMTAB_FORMAL_NAME}[{ARGS_FORMAL_NAME}['{arg_name}']]: {ARRAY_ENTRY_CLASS_NAME}({dtype_arg_name}, {nd_arg_name});\n" + + f"\tref {arg_name} = {arg_name}_array_sym.a;", + [(dtype_arg_name, "type", None), (nd_arg_name, "param", "int")], + ) + + +def unpack_user_symbol(arg_name, symbol_class): + """ + Generate the code to unpack a user-defined symbol from the symbol table. + + - symbol_class should inherit from 'AbstractSymEntry' + - generic user-defined symbol types are not yet supported + + Example: + ``` + var x = st[msgArgs['x']]: MySymbolType; + ``` + """ + return f"\tvar {arg_name} = {SYMTAB_FORMAL_NAME}[{ARGS_FORMAL_NAME}['{arg_name}']]: {symbol_class};" + + +def unpack_scalar_arg(arg_name, arg_type): + """ + Generate the code to unpack a scalar argument + + Example: + ``` + var x = msgArgs['x'].toScalar(int); + ``` + """ + return f"\tvar {arg_name} = {ARGS_FORMAL_NAME}['{arg_name}'].toScalar({arg_type});" + + +def unpack_tuple_arg(arg_name, tuple_size, scalar_type): + """ + Generate the code to unpack a tuple argument + + Example: + ``` + var x = msgArgs['x'].getTuple(int, 2); + ``` + """ + return f"\tvar {arg_name} = {ARGS_FORMAL_NAME}['{arg_name}'].toScalarTuple({scalar_type}, {tuple_size});" + + +def unpack_list_arg(arg_name, arg_type): + """ + Generate the code to unpack a list argument + + Example: + ``` + var x = msgArgs['x'].toScalarList(int); + ``` + """ + + list_elt_type = arg_type.split(",")[1] + return f"\tvar {arg_name} = {ARGS_FORMAL_NAME}['{arg_name}'].toScalarList({list_elt_type});" + + +# TODO: also support generic user-defined symbol types, not just arrays +def gen_signature(user_proc_name, generic_args=None): + """ + Generate the signature for a message handler procedure + + For a concrete command procedure: + ``` + proc ark_reg_(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws { + ``` + + For a generic command procedure: + ``` + proc ark_reg__generic(cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, ): MsgTuple throws { + ``` + + Return the signature and the name of the procedure + """ + if generic_args: + name = "ark_reg_" + user_proc_name + "_generic" + arg_strings = [ + f"{kind} {name}: {ft}" if ft else f"{kind} {name}" + for (name, kind, ft) in generic_args + ] + proc = f"proc {name}(cmd: string, {ARGS_FORMAL_NAME}: {ARGS_FORMAL_TYPE}, {SYMTAB_FORMAL_NAME}: {SYMTAB_FORMAL_TYPE}, {', '.join(arg_strings)}): {RESPONSE_TYPE_NAME} throws {'{'}" + else: + name = "ark_reg_" + user_proc_name + proc = f"proc {name}(cmd: string, {ARGS_FORMAL_NAME}: {ARGS_FORMAL_TYPE}, {SYMTAB_FORMAL_NAME}: {SYMTAB_FORMAL_TYPE}): {RESPONSE_TYPE_NAME} throws {'{'}" + return (proc, name) + + +def gen_arg_unpacking(formals): + """ + Generate argument unpacking code for a message handler procedure + + Returns the chapel code to unpack the arguments, and a list of generic arguments + """ + unpack_lines = [] + generic_args = [] + array_arg_counter = 0 + + for fname, fintent, ftype in formals: + if ftype in chapel_scalar_types: + unpack_lines.append(unpack_scalar_arg(fname, ftype)) + elif ftype == "": + code, array_args = unpack_array_arg(fname, array_arg_counter) + unpack_lines.append(code) + generic_args += array_args + array_arg_counter += 1 + elif "list" in ftype: + unpack_lines.append(unpack_list_arg(fname, ftype)) + elif ftype == "tuple": + print("ERROR: tuple and list types are not yet supported") + continue + else: + # TODO: handle generic user-defined types + unpack_lines.append(unpack_user_symbol(fname, ftype)) + + return ("\n".join(unpack_lines), generic_args) + + +def gen_user_function_call(name, arg_names, mod_name, user_rt): + """ + Generate code to call a user-defined function with the given arguments + + Examples: + ``` + var ark_result = MyModule.myProc(x, y, z); + ``` + + Returns (chapel code, result name) + """ + if user_rt: + return ( + f"\tvar ark_result = {mod_name}.{name}({','.join(arg_names)});", + "ark_result", + ) + else: + return (f"\t{mod_name}.{name}({','.join(arg_names)});", None) + + +def gen_symbol_creation(symbol_class, result_name): + """ + Generate code to create a symbol of the given class from a result + + Example: + ``` + var ark_result_symbol = new shared SymEntry(ark_result); + ``` + + Returns (chapel code, symbol name) + """ + return ( + f"\tvar ark_result_symbol = new shared {symbol_class}({result_name});\n", + "ark_result_symbol", + ) + + +def gen_response(result=None, is_symbol=False): + """ + Generate code to return a response object + + Examples: + ``` + return st.insert(ark_result); + return MsgTuple.fromScalar(x); + return MsgTuple.success(); + ``` + """ + # TODO: need to handle tuple and list return types differently here + if result: + if is_symbol: + return f"\treturn {SYMTAB_FORMAL_NAME}.insert({result});" + else: + return f"\treturn {RESPONSE_TYPE_NAME}.fromScalar({result});" + else: + return "\treturn {RESPONSE_TYPE_NAME}.success();" + + +def gen_command_proc(name, return_type, formals, mod_name): + """ + Generate a chapel command procedure that calls a user-defined procedure + + * name: the name of the user-defined procedure + * return_type: the return type of the user-defined procedure + * formals: a list of tuples in the format (name, storage kind, type expression) + representing the formal parameters of the user-defined procedure + + Returns a tuple of: + * the chapel code for the command procedure + * the name of the command procedure + * a boolean indicating whether the command has generic (param/type) formals + * a list of tuples in the format (name, storage kind, type expression) + representing the generic formals of the command procedure + + proc (cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab, ): MsgTuple throws { + () + + (if necessary) + + } + + """ + + arg_unpack, command_formals = gen_arg_unpacking(formals) + is_generic_command = len(command_formals) > 0 + signature, cmd_name = gen_signature(name, command_formals) + fn_call, result_name = gen_user_function_call( + name, [f[0] for f in formals], mod_name, return_type + ) + + returns_symbol = ( + return_type + and isinstance(return_type, chapel.Identifier) + and return_type.name() not in chapel_scalar_types + ) + returns_array = ( + return_type + and isinstance(return_type, chapel.BracketLoop) + and return_type.is_maybe_array_type() + ) + + if returns_array: + symbol_creation, result_name = gen_symbol_creation( + ARRAY_ENTRY_CLASS_NAME, result_name + ) + else: + symbol_creation = "" + + response = gen_response(result_name, returns_symbol or returns_array) + + command_proc = "\n".join( + [signature, arg_unpack, fn_call, symbol_creation, response, "}"] + ) + + return (command_proc, cmd_name, is_generic_command, command_formals) + + +def stamp_out_command(config, formals, name, cmd_prefix, mod_name, line_num, is_user_proc): + """ + Yield instantiations of a generic command with using the + values from the configuration file + + The instantiated commands will be registered in the command table + + Arguments: + * config: the configuration file + * formals: a list of tuples in the format (name, storage kind, type expression) + representing the generic formals of the command procedure + * name: the name of the generic command procedure + * cmd_prefix: the prefix to use for the command names + * mod_name: the name of the module containing the command procedure + (or the user-defined procedure that the command calls) + + The name of the instantiated command will be in the format: 'cmd_prefix' + where v1, v2, ... are the values of the generic formals + """ + formal_perms = generic_permutations(config, formals) + + for fp in formal_perms: + stamp = stamp_generic_command(name, cmd_prefix, mod_name, fp, line_num, is_user_proc) + yield stamp + + +def register_commands(config, source_files): + """ + Create a chapel source file that registers all the procs annotated with the + 'arkouda.registerCommand' or 'arkouda.instantiateAndRegister' attributes + """ + stamps = [ + "module Commands {", + "use CommandMap, Message, MultiTypeSymbolTable, MultiTypeSymEntry;", + ] + + count = 0 + + for filename, ctx in chapel.files_with_contexts(source_files): + file_stamps = [] + found_annotation = False + + root = ctx.parse(filename)[0] + mod_name = filename.split("/")[-1].split(".")[0] + + stamps.append(f"import {mod_name};") + + # register procs annotated with 'registerCommand', + # creating generic commands and instantiations if necessary + for fn, attr_call in annotated_procs(root, registerAttr): + found_annotation = True + + name = fn.name() + line_num = fn.location().start()[0] + + try: + con_formals, gen_formals = get_formals(fn) + except ValueError as e: + error_message(f"registering '{name}'", e, fn.location()) + continue + + if prefix := attr_call["name"]: + command_prefix = prefix.value() + else: + command_prefix = name + + if len(gen_formals) > 0: + error_message( + f"registering '{name}'", + "generic formals are not allowed in commands", + fn.location(), + ) + continue + + (cmd_proc, cmd_name, is_generic_cmd, cmd_gen_formals) = gen_command_proc( + name, fn.return_type(), con_formals, mod_name + ) + file_stamps.append(cmd_proc) + count += 1; + + if is_generic_cmd > 0: + try: + for stamp in stamp_out_command( + config, + cmd_gen_formals, + cmd_name, + command_prefix, + mod_name, + line_num, + False, + ): + file_stamps.append(stamp) + except ValueError as e: + error_message(f"registering '{name}'", e, fn.location()) + continue + else: + file_stamps.append( + f"registerFunction('{command_prefix}', {cmd_name}, '{mod_name}', {line_num});" + ) + + # instantiate and register procs annotated with 'instantiateAndRegister' + for fn, attr_call in annotated_procs(root, instAndRegisterAttr): + found_annotation = True + + name = fn.name() + line_num = fn.location().start()[0] + + try: + con_formals, gen_formals = get_formals(fn) + except ValueError as e: + error_message(f"registering '{name}'", e, fn.location()) + continue + + if not valid_generic_command_signature(fn, con_formals): + error_message( + f"registering '{name}'", + "generic instantiation of commands must have the signature " + + "'proc (cmd: string, msgArgs: borrowed MessageArgs, st: borrowed SymTab): MsgTuple throws'", + fn.location(), + ) + continue + + if prefix := attr_call["prefix"]: + command_prefix = prefix.value() + else: + command_prefix = name + + try: + for stamp in stamp_out_command( + config, gen_formals, name, command_prefix, mod_name, line_num, True + ): + file_stamps.append(stamp) + count += 1 + except ValueError as e: + error_message(f"registering '{name}'", e, fn.location()) + continue + + if found_annotation: + stamps.extend(file_stamps) + + stamps.append("}") + + return ("\n\n".join(stamps), count) + + +def getModuleFiles(config, src_dir): + with open(config, 'r') as cfg_file: + mods = [] + for line in cfg_file.readlines(): + mod = line.split("#")[0].strip() + if mod != "": + mods.append(f"{mod}.chpl" if mod[0] == '/' else f"{src_dir}/{mod}.chpl") + return mods + + +def main(): + config = json.load(open(sys.argv[1])) + source_files = getModuleFiles(sys.argv[2], sys.argv[3]) + (chpl_src, n) = register_commands(config, source_files) + + with open(sys.argv[3] + "/registry/Commands.chpl", "w") as f: + f.write(chpl_src.replace("\t", " ")) + + print("registered ", n, " commands from ", len(source_files), " modules") + + +if __name__ == "__main__": + main() diff --git a/tests/array_api/binary_ops.py b/tests/array_api/binary_ops.py index a19f46cca7..edc64aca59 100644 --- a/tests/array_api/binary_ops.py +++ b/tests/array_api/binary_ops.py @@ -15,7 +15,7 @@ def test_binops(self): for op in ops: for shape_a, shape_b in zip(SHAPE_A, SHAPE_B): for dtype in ak.ScalarDTypes: - if dtype != ak.bool: + if dtype != ak.bool_: x = xp.asarray(ak.randint(0, 100, shape_a, dtype=dtype, seed=SEED)) y = xp.asarray(ak.randint(0, 100, shape_b, dtype=dtype, seed=SEED)) diff --git a/tests/array_api/sorting.py b/tests/array_api/sorting.py index da83186474..aef09cf621 100644 --- a/tests/array_api/sorting.py +++ b/tests/array_api/sorting.py @@ -42,7 +42,7 @@ def test_argsort(self): def test_sort(self): for shape in SHAPES: for dtype in ak.ScalarDTypes: - if dtype == ak.bool: + if dtype == ak.bool_: continue for axis in range(len(shape)): a = xp.asarray(ak.randint(0, 100, shape, dtype=dtype, seed=SEED)) diff --git a/tests/deprecated/client_test.py b/tests/deprecated/client_test.py index 36ab82c497..1e62bc6186 100644 --- a/tests/deprecated/client_test.py +++ b/tests/deprecated/client_test.py @@ -137,7 +137,7 @@ def test_client_get_server_commands(self): sample of commands. """ cmds = ak.client.get_server_commands() - for cmd in ["connect", "array", "create1D", "tondarray1D", "info", "str"]: + for cmd in ["connect", "create1D", "tondarray1D", "info", "str"]: self.assertTrue(cmd in cmds) def test_client_array_dim_cmd_error(self):