Skip to content

Commit

Permalink
Pull request #10: Port part of the np.array function
Browse files Browse the repository at this point in the history 
Merge in ~STEPAN.SINDELAR_ORACLE.COM/numpy-hpy from ss/array_array to labs-hpy-port

* commit '5222b83f0faa5dd99b1a3c6a659940831c1873a1':
  HPyArray_AssignArray
  Add forgotten capi_warn
  Make PyArray_Descr memcopy work for both legacy/pure layout
  Add HPyType helpers generation for scalars
  Replace h_array_type_global with HPyArray_Type
  HPyArray_SetBaseObject
  HPyDataMem_UserNEW, HPyDataMem_UserNEW_ZEROED
  HPyArray_NewFromDescr_int: check PyCapsule using HPy
  Fully ported HPyArray_NewFromDescr_int (missing: PyArray_SetBaseObject and PyCapsule)
  HPyArray_NewLikeArrayWithShape
  Use HPy Type helpers instead of HPy_AsStruct everywhere
  Fix: use HPyArray_GetNDim
  Call new HPyArray_NewLikeArray
  HPyArray_NewCopy
  Checkpoint: np.array works
  Checkpoint in debugging
  Port np.array to HPy: args parsing
  Set legacy slots of PyCLongDoubleArrType, PyCDoubleArrType, PyCFloatArrType
  • Loading branch information
@fangerer
fangerer committed Mar 21, 2022
2 parents 0e4a5a7 + 5222b83 commit 404a43d
Show file tree
Hide file tree
Showing 28 changed files with 1,255 additions and 210 deletions.
9 changes: 9 additions & 0 deletions numpy/core/include/numpy/ndarrayobject.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ extern "C" {
#define PyArray_DescrCheck(op) PyObject_TypeCheck(op, &PyArrayDescr_Type)
#define HPyArray_DescrCheck(ctx, op) HPy_TypeCheck(ctx, op, HPyArrayDescr_Type)


#define PyArray_Check(op) PyObject_TypeCheck(op, &PyArray_Type)
#define PyArray_CheckExact(op) (((PyObject*)(op))->ob_type == &PyArray_Type)
#define HPyArray_Check(ctx, op) HPy_TypeCheck(ctx, op, HPyArray_Type)
Expand Down Expand Up @@ -184,6 +185,13 @@ PyArray_DiscardWritebackIfCopy(PyArrayObject *arr)
descr = _new_; \
} while(0)

#define HPyArray_DESCR_REPLACE(ctx, descr) do { \
HPy _new_; \
_new_ = HPyArray_DescrNew(ctx, descr); \
HPy_Close(ctx, descr); \
descr = _new_; \
} while(0)

/* Copy should always return contiguous array */
#define PyArray_Copy(obj) PyArray_NewCopy(obj, NPY_CORDER)

Expand Down Expand Up @@ -252,6 +260,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value)
#define NPY_TITLE_KEY(key, value) (NPY_TITLE_KEY_check((key), (value)))

#define DEPRECATE(msg) PyErr_WarnEx(PyExc_DeprecationWarning,msg,1)
#define HPY_DEPRECATE(ctx, msg) HPyErr_WarnEx(ctx,ctx->h_DeprecationWarning,msg,1)
#define DEPRECATE_FUTUREWARNING(msg) PyErr_WarnEx(PyExc_FutureWarning,msg,1)

#ifdef __cplusplus
Expand Down
24 changes: 19 additions & 5 deletions numpy/core/include/numpy/ndarraytypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,9 +613,15 @@ typedef struct {
#define PyDataType_FLAGCHK(dtype, flag) \
(((dtype)->flags & (flag)) == (flag))

#define HPyDataType_FLAGCHK(ctx, dtype, flag) \
((PyArray_Descr_AsStruct(ctx, dtype)->flags & (flag)) == (flag))

#define PyDataType_REFCHK(dtype) \
PyDataType_FLAGCHK(dtype, NPY_ITEM_REFCOUNT)

#define HPyDataType_REFCHK(ctx, dtype) \
HPyDataType_FLAGCHK(ctx, dtype, NPY_ITEM_REFCOUNT)

typedef struct _PyArray_Descr {
PyObject_HEAD
/*
Expand Down Expand Up @@ -1584,13 +1590,19 @@ PyArray_DESCR(const PyArrayObject *arr)
}

static NPY_INLINE HPy
HPyArray_GetBase(HPyContext *ctx, HPy arr)
HPyArray_BASE(HPyContext *ctx, HPy h_arr, PyArrayObject *arr_obj)
{
HPyField f_base = HPyArray_AsFields(ctx, arr)->f_base;
if (f_base._i == 0) {
PyArrayObject_fields *arr = (PyArrayObject_fields*) arr_obj;
if (HPyField_IsNull(arr->f_base)) {
return HPy_NULL;
}
return HPyField_Load(ctx, arr, HPyArray_AsFields(ctx, arr)->f_base);
return HPyField_Load(ctx, h_arr, arr->f_base);
}

static NPY_INLINE HPy
HPyArray_GetBase(HPyContext *ctx, HPy arr)
{
return HPyArray_BASE(ctx, arr, (PyArrayObject*) HPyArray_AsFields(ctx, arr));
}

static NPY_INLINE void
Expand Down Expand Up @@ -1653,7 +1665,7 @@ PyArray_NDIM(const PyArrayObject *arr)
static NPY_INLINE int
HPyArray_GetNDim(HPyContext *ctx, HPy arr)
{
return ((PyArrayObject_fields *) HPy_AsStructLegacy(ctx, arr))->nd;
return ((PyArrayObject_fields *) PyArrayObject_AsStruct(ctx, arr))->nd;
}

static NPY_INLINE void *
Expand Down Expand Up @@ -2070,6 +2082,8 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
void *reserved[3];
} PyArray_DTypeMeta;

HPyType_LEGACY_HELPERS(PyArray_DTypeMeta);

#endif /* NPY_INTERNAL_BUILD */


Expand Down
6 changes: 6 additions & 0 deletions numpy/core/src/common/array_assign.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src,
PyArrayObject *wheremask,
NPY_CASTING casting);


NPY_NO_EXPORT int
HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
HPy h_wheremask,
NPY_CASTING casting);

NPY_NO_EXPORT int
PyArray_AssignRawScalar(PyArrayObject *dst,
PyArray_Descr *src_dtype, char *src_data,
Expand Down
14 changes: 14 additions & 0 deletions numpy/core/src/common/npy_import.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,18 @@ npy_cache_import(const char *module, const char *attr, PyObject **cache)
}
}

// HPY TODO: global cache...
NPY_INLINE static void
npy_hpy_cache_import(HPyContext *ctx, const char *module, const char *attr, HPy *cache)
{
if (NPY_UNLIKELY(HPy_IsNull(*cache))) {
HPy mod = HPyImport_ImportModule(ctx, module);

if (HPy_IsNull(mod)) {
*cache = HPy_GetAttr_s(ctx, mod, attr);
HPy_Close(ctx, mod);
}
}
}

#endif /* NUMPY_CORE_SRC_COMMON_NPY_IMPORT_H_ */
6 changes: 3 additions & 3 deletions numpy/core/src/multiarray/abstractdtypes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ initialize_and_map_pytypes_to_dtypes(HPyContext *ctx)
HPy_Close(ctx, h_PyArray_PyIntAbstractDType);
goto cleanup;
}
PyArray_DTypeMeta *int_abstract_dtype_data = (PyArray_DTypeMeta *) HPy_AsStructLegacy(ctx, h_PyArray_PyIntAbstractDType);
PyArray_DTypeMeta *int_abstract_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_PyArray_PyIntAbstractDType);
int_abstract_dtype_data->dt_slots = &pyintabstractdtype_slots;
int_abstract_dtype_data->flags = NPY_DT_ABSTRACT;
int_abstract_dtype_data->scalar_type = &PyLong_Type;
Expand All @@ -137,7 +137,7 @@ initialize_and_map_pytypes_to_dtypes(HPyContext *ctx)
HPy_Close(ctx, h_PyArray_PyFloatAbstractDType);
goto cleanup;
}
PyArray_DTypeMeta *float_abstract_dtype_data = (PyArray_DTypeMeta *) HPy_AsStructLegacy(ctx, h_PyArray_PyFloatAbstractDType);
PyArray_DTypeMeta *float_abstract_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_PyArray_PyFloatAbstractDType);
float_abstract_dtype_data->dt_slots = &pyfloatabstractdtype_slots;
float_abstract_dtype_data->flags = NPY_DT_ABSTRACT;
float_abstract_dtype_data->scalar_type = &PyFloat_Type;
Expand All @@ -149,7 +149,7 @@ initialize_and_map_pytypes_to_dtypes(HPyContext *ctx)
HPy_Close(ctx, h_PyArray_PyComplexAbstractDType);
goto cleanup;
}
PyArray_DTypeMeta *complex_abstract_dtype_data = (PyArray_DTypeMeta *) HPy_AsStructLegacy(ctx, h_PyArray_PyComplexAbstractDType);
PyArray_DTypeMeta *complex_abstract_dtype_data = PyArray_DTypeMeta_AsStruct(ctx, h_PyArray_PyComplexAbstractDType);
complex_abstract_dtype_data->dt_slots = &pycomplexabstractdtype_slots;
complex_abstract_dtype_data->flags = NPY_DT_ABSTRACT;
complex_abstract_dtype_data->scalar_type = &PyComplex_Type;
Expand Down
143 changes: 82 additions & 61 deletions numpy/core/src/multiarray/alloc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ PyDataMem_Handler default_handler = {
PyObject *PyDataMem_DefaultHandler;

#if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
PyObject *current_handler;
HPy current_handler;
#endif

int uo_index=0; /* user_override index */
Expand All @@ -416,15 +416,26 @@ int uo_index=0; /* user_override index */

NPY_NO_EXPORT void *
PyDataMem_UserNEW(size_t size, PyObject *mem_handler)
{
HPyContext *ctx = npy_get_context();
HPy h = HPy_FromPyObject(ctx, mem_handler);
void *res = HPyDataMem_UserNEW(ctx, size, h);
HPy_Close(ctx, h);
return res;
}

NPY_NO_EXPORT void *
HPyDataMem_UserNEW(HPyContext *ctx, size_t size, HPy mem_handler)
{
void *result;
PyDataMem_Handler *handler = (PyDataMem_Handler *) PyCapsule_GetPointer(mem_handler, "mem_handler");
PyDataMem_Handler *handler = (PyDataMem_Handler *) HPyCapsule_GetPointer(ctx, mem_handler, "mem_handler");
if (handler == NULL) {
return NULL;
}
assert(size != 0);
result = handler->allocator.malloc(handler->allocator.ctx, size);
if (_PyDataMem_eventhook != NULL) {
capi_warn("HPyDataMem_UserNEW_ZEROED: GIL usage if _PyDataMem_eventhook is set");
NPY_ALLOW_C_API_DEF
NPY_ALLOW_C_API
if (_PyDataMem_eventhook != NULL) {
Expand All @@ -439,14 +450,25 @@ PyDataMem_UserNEW(size_t size, PyObject *mem_handler)

NPY_NO_EXPORT void *
PyDataMem_UserNEW_ZEROED(size_t nmemb, size_t size, PyObject *mem_handler)
{
HPyContext *ctx = npy_get_context();
HPy h = HPy_FromPyObject(ctx, mem_handler);
void *res = HPyDataMem_UserNEW_ZEROED(ctx, nmemb, size, h);
HPy_Close(ctx, h);
return res;
}

NPY_NO_EXPORT void *
HPyDataMem_UserNEW_ZEROED(HPyContext *ctx, size_t nmemb, size_t size, HPy mem_handler)
{
void *result;
PyDataMem_Handler *handler = (PyDataMem_Handler *) PyCapsule_GetPointer(mem_handler, "mem_handler");
PyDataMem_Handler *handler = (PyDataMem_Handler *) HPyCapsule_GetPointer(ctx, mem_handler, "mem_handler");
if (handler == NULL) {
return NULL;
}
result = handler->allocator.calloc(handler->allocator.ctx, nmemb, size);
if (_PyDataMem_eventhook != NULL) {
capi_warn("HPyDataMem_UserNEW_ZEROED: GIL usage if _PyDataMem_eventhook is set");
NPY_ALLOW_C_API_DEF
NPY_ALLOW_C_API
if (_PyDataMem_eventhook != NULL) {
Expand Down Expand Up @@ -519,77 +541,76 @@ PyDataMem_UserRENEW(void *ptr, size_t size, PyObject *mem_handler)
NPY_NO_EXPORT PyObject *
PyDataMem_SetHandler(PyObject *handler)
{
PyObject *old_handler;
#if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
PyObject *token;
if (PyContextVar_Get(current_handler, NULL, &old_handler)) {
return NULL;
}
if (handler == NULL) {
handler = PyDataMem_DefaultHandler;
}
token = PyContextVar_Set(current_handler, handler);
if (token == NULL) {
Py_DECREF(old_handler);
return NULL;
}
Py_DECREF(token);
return old_handler;
#else
PyObject *p;
p = PyThreadState_GetDict();
if (p == NULL) {
return NULL;
}
old_handler = PyDict_GetItemString(p, "current_allocator");
if (old_handler == NULL) {
old_handler = PyDataMem_DefaultHandler
}
Py_INCREF(old_handler);
if (handler == NULL) {
handler = PyDataMem_DefaultHandler;
}
const int error = PyDict_SetItemString(p, "current_allocator", handler);
if (error) {
Py_DECREF(old_handler);
return NULL;
}
return old_handler;
#endif
hpy_abort_not_implemented("PyDataMem_SetHandler");
// PyObject *old_handler;
// #if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
// PyObject *token;
// if (PyContextVar_Get(current_handler, NULL, &old_handler)) {
// return NULL;
// }
// if (handler == NULL) {
// handler = PyDataMem_DefaultHandler;
// }
// token = PyContextVar_Set(current_handler, handler);
// if (token == NULL) {
// Py_DECREF(old_handler);
// return NULL;
// }
// Py_DECREF(token);
// return old_handler;
// #else
// PyObject *p;
// p = PyThreadState_GetDict();
// if (p == NULL) {
// return NULL;
// }
// old_handler = PyDict_GetItemString(p, "current_allocator");
// if (old_handler == NULL) {
// old_handler = PyDataMem_DefaultHandler
// }
// Py_INCREF(old_handler);
// if (handler == NULL) {
// handler = PyDataMem_DefaultHandler;
// }
// const int error = PyDict_SetItemString(p, "current_allocator", handler);
// if (error) {
// Py_DECREF(old_handler);
// return NULL;
// }
// return old_handler;
// #endif
}

/*NUMPY_API
/*
* Return the policy that will be used to allocate data
* for the next PyArrayObject. On failure, return NULL.
*/
NPY_NO_EXPORT PyObject *
PyDataMem_GetHandler()
NPY_NO_EXPORT HPy
HPyDataMem_GetHandler(HPyContext *ctx)
{
PyObject *handler;
HPy handler;
#if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
if (PyContextVar_Get(current_handler, NULL, &handler)) {
return NULL;
if (HPyContextVar_Get(ctx, current_handler, HPy_NULL, &handler)) {
return HPy_NULL;
}
return handler;
#else
PyObject *p = PyThreadState_GetDict();
if (p == NULL) {
return NULL;
}
handler = PyDict_GetItemString(p, "current_allocator");
if (handler == NULL) {
handler = PyCapsule_New(&default_handler, "mem_handler", NULL);
if (handler == NULL) {
return NULL;
}
}
else {
Py_INCREF(handler);
}
return handler;
hpy_abort_not_implemented("HPyDataMem_GetHandler on older Pypy versions");
#endif
}

/*NUMPY_API
*/
NPY_NO_EXPORT PyObject *
PyDataMem_GetHandler()
{
HPyContext *ctx = npy_get_context();
HPy h = HPyDataMem_GetHandler(ctx);
PyObject *res = HPy_AsPyObject(ctx, h);
HPy_Close(ctx, h);
return res;
}

NPY_NO_EXPORT PyObject *
get_handler_name(PyObject *NPY_UNUSED(self), PyObject *args)
{
Expand Down
11 changes: 10 additions & 1 deletion numpy/core/src/multiarray/alloc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ PyDataMem_UserNEW(npy_uintp sz, PyObject *mem_handler);
NPY_NO_EXPORT void *
PyDataMem_UserNEW_ZEROED(size_t nmemb, size_t size, PyObject *mem_handler);

NPY_NO_EXPORT void *
HPyDataMem_UserNEW(HPyContext *ctx, size_t size, HPy mem_handler);

NPY_NO_EXPORT void *
HPyDataMem_UserNEW_ZEROED(HPyContext *ctx, size_t nmemb, size_t size, HPy mem_handler);

NPY_NO_EXPORT void
PyDataMem_UserFREE(void * p, npy_uintp sd, PyObject *mem_handler);

Expand All @@ -42,12 +48,15 @@ npy_free_cache_dim_array(PyArrayObject * arr)

extern PyDataMem_Handler default_handler;
#if (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x07030600)
extern PyObject *current_handler; /* PyContextVar/PyCapsule */
extern HPy current_handler; /* PyContextVar/PyCapsule */
#endif

NPY_NO_EXPORT PyObject *
get_handler_name(PyObject *NPY_UNUSED(self), PyObject *obj);
NPY_NO_EXPORT PyObject *
get_handler_version(PyObject *NPY_UNUSED(self), PyObject *obj);

NPY_NO_EXPORT HPy
HPyDataMem_GetHandler(HPyContext *ctx);

#endif /* NUMPY_CORE_SRC_MULTIARRAY_ALLOC_H_ */
Loading

0 comments on commit 404a43d

Please sign in to comment.