From 577ee1638804dcc951e973e06bd1220200e6e34f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 09:00:18 +1000 Subject: [PATCH 001/273] update to latest versions --- .gitmodules | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7c5c5312a..83c03de48 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,11 @@ [submodule "Python"] path = Python url = https://github.com/python/cpython.git -[submodule "CoreCLR"] + branch = 3.9 +[submodule "dotnet"] path = CoreCLR - url = https://github.com/dotnet/coreclr.git - branch = release/1.0.0-rc2 + url = https://github.com/dotnet/runtime + branch = release/5.0 [submodule "Tests/Catch"] path = Tests/Catch url = https://github.com/philsquared/Catch.git From c32da6bad0fd26652d6dc90f5f4ae705a83755de Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 09:20:12 +1000 Subject: [PATCH 002/273] update refs --- CoreCLR | 2 +- Python | 2 +- Tests/Catch | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CoreCLR b/CoreCLR index c6de64f76..fa2f1fb4f 160000 --- a/CoreCLR +++ b/CoreCLR @@ -1 +1 @@ -Subproject commit c6de64f76ca47be6ff97c12067011429a34ef7cd +Subproject commit fa2f1fb4fe0962226e9454d12214435c82a34dbc diff --git a/Python b/Python index 2773add19..bdf46bc7e 160000 --- a/Python +++ b/Python @@ -1 +1 @@ -Subproject commit 2773add19aff873377d81e3bb6ab8aa942756f5a +Subproject commit bdf46bc7e12c08aaab676a2c947f03daffe035a7 diff --git a/Tests/Catch b/Tests/Catch index 3bd20bf2c..fba0feeba 160000 --- a/Tests/Catch +++ b/Tests/Catch @@ -1 +1 @@ -Subproject commit 3bd20bf2cd05cd88e767ec1aeb856bf8032a04c7 +Subproject commit fba0feeba1e3b94b846d4c64338d53356671d44a From 3f8724b91bbb9d5373facedabf876f5276b737e7 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 10:36:21 +1000 Subject: [PATCH 003/273] Use the correct Python lib path --- Pyjion/absvalue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pyjion/absvalue.h b/Pyjion/absvalue.h index f2fd8fa42..a3646a5ed 100644 --- a/Pyjion/absvalue.h +++ b/Pyjion/absvalue.h @@ -26,7 +26,7 @@ #ifndef ABSVALUE_H #define ABSVALUE_H -#include +#include #include #include "cowvector.h" From c40d4f3473518ecca6f6069a7a9ae24039dc8cc5 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 10:36:49 +1000 Subject: [PATCH 004/273] Fix invalid token pasting error --- Pyjion/taggedptr.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Pyjion/taggedptr.h b/Pyjion/taggedptr.h index 364d010ee..f72ff1251 100644 --- a/Pyjion/taggedptr.h +++ b/Pyjion/taggedptr.h @@ -57,7 +57,5 @@ inline bool can_tag(tagged_ptr value) { #define NUMBER_SIZE (((sizeof(PyVarObject) + sizeof(PY_UINT32_T) * DIGITS_IN_TAGGED_PTR) / sizeof(size_t)) + sizeof(size_t)) #define INIT_TMP_NUMBER(name, value) \ size_t tmp_##name[NUMBER_SIZE]; \ - PyObject* ##name = init_number(tmp_##name, value); - - + PyObject* name = init_number(tmp_##name, value); #endif \ No newline at end of file From 4427ca67704a1b37a2b1c2a4ce14be06c11ab348 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 10:37:38 +1000 Subject: [PATCH 005/273] Avoid strict casting errors, Add simple clang floor division option, Fix goto-over-initialized Fix invalid PyThreadState field names --- Pyjion/intrins.cpp | 62 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Pyjion/intrins.cpp b/Pyjion/intrins.cpp index 5060269d0..a20a33ff1 100644 --- a/Pyjion/intrins.cpp +++ b/Pyjion/intrins.cpp @@ -168,11 +168,11 @@ int PyJit_RichEquals_Long(PyObject *left, PyObject *right, void**state) { int PyJit_RichEquals_Generic(PyObject *left, PyObject *right, void**state) { if (left->ob_type == right->ob_type) { if (PyUnicode_CheckExact(left)) { - *state = &PyJit_RichEquals_Str; + *state = (void*)&PyJit_RichEquals_Str; return PyJit_RichEquals_Str(left, right, state); } else if (PyLong_CheckExact(left)) { - *state = &PyJit_RichEquals_Long; + *state = (void*)&PyJit_RichEquals_Long; return PyJit_RichEquals_Long(left, right, state); } } @@ -554,15 +554,15 @@ void PyJit_PrepareException(PyObject** exc, PyObject**val, PyObject** tb, PyObje auto tstate = PyThreadState_GET(); // we take ownership of these into locals... - if (tstate->exc_type != nullptr) { - *oldexc = tstate->exc_type; + if (tstate->curexc_type != nullptr) { + *oldexc = tstate->curexc_type; } else { *oldexc = Py_None; Py_INCREF(Py_None); } - *oldVal = tstate->exc_value; - *oldTb = tstate->exc_traceback; + *oldVal = tstate->curexc_value; + *oldTb = tstate->curexc_traceback; PyErr_Fetch(exc, val, tb); /* Make the raw exception data @@ -576,11 +576,11 @@ void PyJit_PrepareException(PyObject** exc, PyObject**val, PyObject** tb, PyObje else PyException_SetTraceback(*val, Py_None); Py_INCREF(*exc); - tstate->exc_type = *exc; + tstate->curexc_type = *exc; Py_INCREF(*val); - tstate->exc_value = *val; + tstate->curexc_value = *val; assert(PyExceptionInstance_Check(*val)); - tstate->exc_traceback = *tb; + tstate->curexc_traceback = *tb; if (*tb == NULL) *tb = Py_None; Py_INCREF(*tb); @@ -589,12 +589,12 @@ void PyJit_PrepareException(PyObject** exc, PyObject**val, PyObject** tb, PyObje void PyJit_UnwindEh(PyObject*exc, PyObject*val, PyObject*tb) { auto tstate = PyThreadState_GET(); assert(val == nullptr || PyExceptionInstance_Check(val)); - auto oldtb = tstate->exc_traceback; - auto oldtype = tstate->exc_type; - auto oldvalue = tstate->exc_value; - tstate->exc_traceback = tb; - tstate->exc_type = exc; - tstate->exc_value = val; + auto oldtb = tstate->curexc_traceback; + auto oldtype = tstate->curexc_type; + auto oldvalue = tstate->curexc_value; + tstate->curexc_traceback = tb; + tstate->curexc_type = exc; + tstate->curexc_value = val; Py_XDECREF(oldtb); Py_XDECREF(oldtype); Py_XDECREF(oldvalue); @@ -1010,9 +1010,9 @@ int PyJit_Raise(PyObject *exc, PyObject *cause) { /* Reraise */ PyThreadState *tstate = PyThreadState_GET(); PyObject *tb; - type = tstate->exc_type; - value = tstate->exc_value; - tb = tstate->exc_traceback; + type = tstate->curexc_type; + value = tstate->curexc_value; + tb = tstate->curexc_traceback; if (type == Py_None || type == nullptr) { PyErr_SetString(PyExc_RuntimeError, "No active exception to reraise"); @@ -1437,12 +1437,14 @@ PyObject** PyJit_UnpackSequenceEx(PyObject* seq, size_t leftSize, size_t rightSi // the function allocated space on the stack for us to // store these temporarily. auto it = PyObject_GetIter(seq); + int i = 0; + auto sp = tempStorage + leftSize + rightSize; + if (it == nullptr) { goto Error; } - auto sp = tempStorage + leftSize + rightSize; - int i = 0; + for (; i < leftSize; i++) { auto w = PyIter_Next(it); if (w == NULL) { @@ -1808,15 +1810,15 @@ PyObject* Call0_CFunction(PyObject *target, void** addr) { PyObject* Call0_Generic(PyObject *target, void** addr) { PyObject* res; if (PyFunction_Check(target)) { - *addr = &Call0_Function; + *addr = (void*)&Call0_Function; return Call0_Function(target, addr); } else if (PyCFunction_Check(target)) { - *addr = Call0_CFunction; + *addr = (void*)Call0_CFunction; return Call0_CFunction(target, addr); } else if (PyMethod_Check(target) && PyMethod_GET_SELF(target) != NULL) { - *addr = Call0_Method; + *addr = (void*)Call0_Method; return Call0_Method(target, addr); } else { @@ -1829,6 +1831,7 @@ PyObject* Call0_Generic(PyObject *target, void** addr) { PyObject* Call1(PyObject *target, PyObject* arg0) { PyObject* res = nullptr; + auto args = PyTuple_New(1); if (PyFunction_Check(target)) { PyObject* stack[1] = { arg0 }; res = fast_function(target, stack, 1); @@ -1844,8 +1847,6 @@ PyObject* Call1(PyObject *target, PyObject* arg0) { goto error; } - - auto args = PyTuple_New(1); if (args == nullptr) { Py_DECREF(arg0); goto error; @@ -1865,6 +1866,7 @@ PyObject* Call1(PyObject *target, PyObject* arg0) { PyObject* Call2(PyObject *target, PyObject* arg0, PyObject* arg1) { PyObject* res = nullptr; + auto args = PyTuple_New(2); if (PyFunction_Check(target)) { PyObject* stack[2] = { arg0, arg1 }; res = fast_function(target, stack, 2); @@ -1881,8 +1883,6 @@ PyObject* Call2(PyObject *target, PyObject* arg0, PyObject* arg1) { goto error; } - - auto args = PyTuple_New(2); if (args == nullptr) { Py_DECREF(arg0); Py_DECREF(arg1); @@ -1904,6 +1904,7 @@ PyObject* Call2(PyObject *target, PyObject* arg0, PyObject* arg1) { PyObject* Call3(PyObject *target, PyObject* arg0, PyObject* arg1, PyObject* arg2) { PyObject* res = nullptr; + auto args = PyTuple_New(3); if (PyFunction_Check(target)) { PyObject* stack[3] = { arg0, arg1, arg2 }; res = fast_function(target, stack, 3); @@ -1921,7 +1922,6 @@ PyObject* Call3(PyObject *target, PyObject* arg0, PyObject* arg1, PyObject* arg2 goto error; } - auto args = PyTuple_New(3); if (args == nullptr) { Py_DECREF(arg0); Py_DECREF(arg1); @@ -1945,6 +1945,7 @@ PyObject* Call3(PyObject *target, PyObject* arg0, PyObject* arg1, PyObject* arg2 PyObject* Call4(PyObject *target, PyObject* arg0, PyObject* arg1, PyObject* arg2, PyObject* arg3) { PyObject* res = nullptr; + auto args = PyTuple_New(4); if (PyFunction_Check(target)) { PyObject* stack[4] = { arg0, arg1, arg2, arg3 }; res = fast_function(target, stack, 4); @@ -1955,7 +1956,6 @@ PyObject* Call4(PyObject *target, PyObject* arg0, PyObject* arg1, PyObject* arg2 goto error; } - auto args = PyTuple_New(4); if (args == nullptr) { Py_DECREF(arg0); Py_DECREF(arg1); @@ -2210,7 +2210,7 @@ inline PyObject* PyJit_Tagged_Modulo(tagged_ptr left, tagged_ptr right) { #ifdef _MSC_VER if (right > 0 && left > 0 && SafeModulus(left, right, res)) { #else -#error No support for this compiler + if (right > 0 && left > 0 && left % right) { #endif if (can_tag(res)) { return TAG_IT(res); @@ -2245,7 +2245,7 @@ inline PyObject* PyJit_Tagged_FloorDivide(tagged_ptr left, tagged_ptr right) { #ifdef _MSC_VER if (SafeDivide(left, right, res)) { #else -#error No support for this compiler + if (floor(left / right)){ #endif if (can_tag(res)) { return TAG_IT(res); From 2c12859975067c9d2d088fd8e913601657ff29ef Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 10:38:03 +1000 Subject: [PATCH 006/273] Add a build option for *nix Create a CMakeLists.txt file for CMake building --- BuildDebugPython.sh | 5 +++++ CMakeLists.txt | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 BuildDebugPython.sh create mode 100644 CMakeLists.txt diff --git a/BuildDebugPython.sh b/BuildDebugPython.sh new file mode 100644 index 000000000..d7f56fac3 --- /dev/null +++ b/BuildDebugPython.sh @@ -0,0 +1,5 @@ +echo Building CPython in debug mode for x64 ... +cd Python/ +./configure --with-pydebug +make -s -j2 +cd .. \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..d3f180e0d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.2) +project(pyjion) + +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD_REQUIRED) + + +find_package (Python3 COMPONENTS Interpreter Development) +include_directories(${Python3_INCLUDE_DIRS}) + +set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) +add_library(pyjion SHARED ${SOURCES}) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -Wno-invalid-token-paste") + +option(WERROR "Turn warnings into errors" OFF) +if (WERROR) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") +endif () \ No newline at end of file From 6518b0c1f5e6e54ecda8b71e792041b918b8dc77 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 11:08:55 +1000 Subject: [PATCH 007/273] Link the CoreCLR cmake project as a dependency Fixup some C++ compliance issues --- CMakeLists.txt | 10 +++++++++- Pyjion/cowvector.h | 16 ++++++---------- Pyjion/jitinfo.h | 11 +++++++++-- Pyjion/pycomp.h | 12 ++++++++++-- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3f180e0d..6f7ec7da7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,10 +8,18 @@ set (CMAKE_CXX_STANDARD_REQUIRED) find_package (Python3 COMPONENTS Interpreter Development) include_directories(${Python3_INCLUDE_DIRS}) +include(CoreCLR/eng/native/functions.cmake) +include(CoreCLR/src/coreclr/pgosupport.cmake) +set(CLR_CMAKE_TARGET_ARCH_AMD64 1) + +add_subdirectory(CoreCLR/src/coreclr/src/jit) +include_directories(CoreCLR/src/coreclr/src/pal/prebuilt/inc) +include_directories(CoreCLR/src/coreclr/src/inc) + set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) add_library(pyjion SHARED ${SOURCES}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -Wno-invalid-token-paste") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare") option(WERROR "Turn warnings into errors" OFF) if (WERROR) diff --git a/Pyjion/cowvector.h b/Pyjion/cowvector.h index f894d9acd..f650067ad 100644 --- a/Pyjion/cowvector.h +++ b/Pyjion/cowvector.h @@ -36,13 +36,12 @@ using namespace std; template class CowData { shared_ptr m_data; public: - CowData() { - } + CowData() = default; - CowData(T *data) : m_data(data) { + explicit CowData(T *data) : m_data(data) { } - CowData(shared_ptr data) : m_data(data) { + explicit CowData(shared_ptr data) : m_data(data) { } bool operator ==(CowData other) { @@ -72,10 +71,9 @@ template class CowData { // Copy on write vector implementation template class CowVector : public CowData> { public: - CowVector() { - } + CowVector() = default; - CowVector(size_t size) : CowData(new vector(size)) { + explicit CowVector(size_t size) : CowData>(new vector(size)) { } T operator[](size_t index) { @@ -109,9 +107,7 @@ template class CowSet : public CowData> { typedef typename unordered_set::iterator iterator; static shared_ptr> s_empty; public: - CowSet() : CowData(get_empty()) { - - } + CowSet() : CowData>(get_empty()) { } ; iterator find(const key_type& k) { return get_current().find(k); diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 57594c7ae..40bee16db 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -30,7 +30,15 @@ #define FEATURE_NO_HOST #define USE_STL #include + +#ifdef _WIN32 #include +#include +#include +#else +// TODO intrinsics from x86/LLVM +#endif + #include #include #include @@ -38,9 +46,8 @@ #include #include #include -#include + #include -#include #include #include diff --git a/Pyjion/pycomp.h b/Pyjion/pycomp.h index abd81be5c..540dffc61 100644 --- a/Pyjion/pycomp.h +++ b/Pyjion/pycomp.h @@ -27,7 +27,15 @@ #define PYCOMP_H #include + +#ifdef _WIN32 #include +#include +#include +#else +// TODO intrinsics from x86/LLVM +#endif + #include #include #include @@ -35,9 +43,9 @@ #include #include #include -#include + #include -#include + #include #include From 77c60704ca66d546679fa03def3cf8bafe34956f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 12:11:23 +1000 Subject: [PATCH 008/273] Update imports for PAL/CoreCLR Windows API shims --- CMakeLists.txt | 17 ++++++++++++----- Pyjion/absint.h | 11 +++++++++++ Pyjion/cowvector.h | 22 +++++++++++----------- Pyjion/pycomp.cpp | 2 ++ Pyjion/pycomp.h | 9 ++------- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f7ec7da7..6ade40279 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,11 +8,18 @@ set (CMAKE_CXX_STANDARD_REQUIRED) find_package (Python3 COMPONENTS Interpreter Development) include_directories(${Python3_INCLUDE_DIRS}) -include(CoreCLR/eng/native/functions.cmake) -include(CoreCLR/src/coreclr/pgosupport.cmake) -set(CLR_CMAKE_TARGET_ARCH_AMD64 1) - -add_subdirectory(CoreCLR/src/coreclr/src/jit) +#set (CLR_DIR CoreCLR/src/coreclr) +# +#include(CoreCLR/eng/native/configurecompiler.cmake) +#include(CoreCLR/eng/native/configureplatform.cmake) +#include(CoreCLR/eng/native/functions.cmake) +#include(CoreCLR/src/coreclr/pgosupport.cmake) +#set(CLR_CMAKE_TARGET_ARCH_AMD64 1) + +#add_subdirectory(CoreCLR/src/coreclr/src/jit) +#add_subdirectory(CoreCLR/src/coreclr/src/pal) +include_directories(CoreCLR/src/coreclr/src/pal/inc) +include_directories(CoreCLR/src/coreclr/src/pal/inc/rt) include_directories(CoreCLR/src/coreclr/src/pal/prebuilt/inc) include_directories(CoreCLR/src/coreclr/src/inc) diff --git a/Pyjion/absint.h b/Pyjion/absint.h index 171a7356d..391e4eceb 100644 --- a/Pyjion/absint.h +++ b/Pyjion/absint.h @@ -26,6 +26,13 @@ #ifndef ABSINT_H #define ABSINT_H +#include + +#ifndef _WIN32 +#define _ASSERTE(cond) \ + assert(cond); +#endif + #include #include #include @@ -154,8 +161,12 @@ struct BlockInfo { }; +#ifdef _WIN32 class __declspec(dllexport) AbstractInterpreter { #pragma warning (disable:4251) +#else + class AbstractInterpreter { +#endif // ** Results produced: // Tracks the interpreter state before each opcode unordered_map m_startStates; diff --git a/Pyjion/cowvector.h b/Pyjion/cowvector.h index f650067ad..3da3f2f6a 100644 --- a/Pyjion/cowvector.h +++ b/Pyjion/cowvector.h @@ -77,27 +77,27 @@ template class CowVector : public CowData> { } T operator[](size_t index) { - return get_current()[index]; + return this->get_current()[index]; } size_t size() { - return get_current().size(); + return this->get_current().size(); } void replace(size_t index, T value) { - get_mutable()[index] = value; + this->get_mutable()[index] = value; } T& back() { - return get_current().back(); + return this->get_current().back(); } void push_back(T value) { - get_mutable().push_back(value); + this->get_mutable().push_back(value); } void pop_back() { - get_mutable().pop_back(); + this->get_mutable().pop_back(); } }; @@ -110,22 +110,22 @@ template class CowSet : public CowData> { CowSet() : CowData>(get_empty()) { } ; iterator find(const key_type& k) { - return get_current().find(k); + return this->get_current().find(k); } size_t size() { - return get_current().size(); + return this->get_current().size(); } void insert(const value_type& val) { - get_mutable().insert(val); + this->get_mutable().insert(val); } iterator begin() { - return get_current().begin(); + return this->get_current().begin(); } iterator end() { - return get_current().end(); + return this->get_current().end(); } CowSet combine(CowSet other) { diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 29d9a1242..8e89d15fb 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -23,6 +23,8 @@ * */ +#include + #include "pycomp.h" #include #include diff --git a/Pyjion/pycomp.h b/Pyjion/pycomp.h index 540dffc61..c4b3c2a70 100644 --- a/Pyjion/pycomp.h +++ b/Pyjion/pycomp.h @@ -28,18 +28,13 @@ #include -#ifdef _WIN32 #include -#include -#include -#else -// TODO intrinsics from x86/LLVM -#endif - #include #include #include #include +#include +#include #include #include #include From 1170c22a1076ee536ef3df7ceedd21d7eddd970a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 12:54:47 +1000 Subject: [PATCH 009/273] Use RT/PAL for header --- Pyjion/absint.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Pyjion/absint.h b/Pyjion/absint.h index 391e4eceb..1f2dcddd7 100644 --- a/Pyjion/absint.h +++ b/Pyjion/absint.h @@ -26,11 +26,10 @@ #ifndef ABSINT_H #define ABSINT_H -#include +#include #ifndef _WIN32 -#define _ASSERTE(cond) \ - assert(cond); +#include #endif #include From f48ac8b408733f60cc31ee5ed06c4648b46b1f09 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 12:55:07 +1000 Subject: [PATCH 010/273] More PAL/Compat updates --- Pyjion/intrins.cpp | 2 +- Pyjion/jitinfo.h | 27 +++++---------------------- Pyjion/jitinit.h | 1 - Pyjion/pycomp.cpp | 6 ++++-- Pyjion/pycomp.h | 13 +------------ Pyjion/pyjit.h | 1 - 6 files changed, 11 insertions(+), 39 deletions(-) diff --git a/Pyjion/intrins.cpp b/Pyjion/intrins.cpp index a20a33ff1..c2c74b50d 100644 --- a/Pyjion/intrins.cpp +++ b/Pyjion/intrins.cpp @@ -27,7 +27,7 @@ #include "intrins.h" #include "taggedptr.h" -#include +#include #ifdef _MSC_VER diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 40bee16db..dc1bb6170 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -28,37 +28,20 @@ #define FEATURE_NO_HOST -#define USE_STL -#include - -#ifdef _WIN32 -#include -#include -#include -#else -// TODO intrinsics from x86/LLVM -#endif - -#include -#include -#include -#include -#include -#include -#include - -#include #include -#include +//#include #include #include +#include +#include +#include #include #include #include -#include + #include #include "codemodel.h" diff --git a/Pyjion/jitinit.h b/Pyjion/jitinit.h index 7e3e402c2..92ff314d6 100644 --- a/Pyjion/jitinit.h +++ b/Pyjion/jitinit.h @@ -14,7 +14,6 @@ #include #include #include -#include #include #include diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 8e89d15fb..b9c34df1a 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -23,9 +23,11 @@ * */ -#include - +#include +#define PAL_STDCPP_COMPAT 1 #include "pycomp.h" + +#include #include #include diff --git a/Pyjion/pycomp.h b/Pyjion/pycomp.h index c4b3c2a70..d7b182a7c 100644 --- a/Pyjion/pycomp.h +++ b/Pyjion/pycomp.h @@ -26,25 +26,14 @@ #ifndef PYCOMP_H #define PYCOMP_H -#include #include -#include -#include -#include -#include + #include #include #include -#include #include -#include - - -#include -#include - #include "ipycomp.h" #include "jitinfo.h" #include "codemodel.h" diff --git a/Pyjion/pyjit.h b/Pyjion/pyjit.h index b9fc33b07..111d2fc96 100644 --- a/Pyjion/pyjit.h +++ b/Pyjion/pyjit.h @@ -38,7 +38,6 @@ #include #include #include -#include #include #include From abe22c11464f99671f8eea9e15b7bf3d33b56209 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 12:57:37 +1000 Subject: [PATCH 011/273] Remove deprecated COR flag --- Pyjion/codemodel.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pyjion/codemodel.h b/Pyjion/codemodel.h index be144f87e..7a919c395 100644 --- a/Pyjion/codemodel.h +++ b/Pyjion/codemodel.h @@ -98,7 +98,7 @@ class BaseMethod { virtual void get_call_info(CORINFO_CALL_INFO *pResult) = 0; virtual DWORD get_method_attrs() { - return CORINFO_FLG_NOSECURITYWRAP | CORINFO_FLG_STATIC | CORINFO_FLG_NATIVE; + return CORINFO_FLG_STATIC | CORINFO_FLG_NATIVE; } virtual void findSig(CORINFO_SIG_INFO *sig) = 0; virtual void* get_addr() = 0; @@ -206,7 +206,7 @@ class VirtualMethod : public Method { } virtual DWORD get_method_attrs() { - return CORINFO_FLG_NOSECURITYWRAP | CORINFO_FLG_VIRTUAL | CORINFO_FLG_NATIVE; + return CORINFO_FLG_VIRTUAL | CORINFO_FLG_NATIVE; } }; From 553b8fcf92320f959667f9105f1382c91d6c27ea Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 14:38:26 +1000 Subject: [PATCH 012/273] More portability updates to support the new Dotnet CLR build process --- CMakeLists.txt | 54 +++-- Pyjion/absint.cpp | 528 +++++++++-------------------------------- Pyjion/absint.h | 10 +- Pyjion/absvalue.h | 1 + Pyjion/cee.h | 10 +- Pyjion/codemodel.h | 2 +- Pyjion/compat/malloc.h | 1 + Pyjion/intrins.cpp | 3 +- Pyjion/jitinfo.h | 1 - Pyjion/jitinit.h | 2 +- Pyjion/pycomp.cpp | 6 +- 11 files changed, 159 insertions(+), 459 deletions(-) create mode 100644 Pyjion/compat/malloc.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ade40279..59a1638e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,34 +1,44 @@ cmake_minimum_required(VERSION 3.2) project(pyjion) -set (CMAKE_CXX_STANDARD 11) -set (CMAKE_CXX_STANDARD_REQUIRED) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +include(CheckCXXCompilerFlag) find_package (Python3 COMPONENTS Interpreter Development) include_directories(${Python3_INCLUDE_DIRS}) -#set (CLR_DIR CoreCLR/src/coreclr) -# -#include(CoreCLR/eng/native/configurecompiler.cmake) -#include(CoreCLR/eng/native/configureplatform.cmake) -#include(CoreCLR/eng/native/functions.cmake) -#include(CoreCLR/src/coreclr/pgosupport.cmake) -#set(CLR_CMAKE_TARGET_ARCH_AMD64 1) - -#add_subdirectory(CoreCLR/src/coreclr/src/jit) -#add_subdirectory(CoreCLR/src/coreclr/src/pal) -include_directories(CoreCLR/src/coreclr/src/pal/inc) -include_directories(CoreCLR/src/coreclr/src/pal/inc/rt) -include_directories(CoreCLR/src/coreclr/src/pal/prebuilt/inc) +set (CLR_DIR CoreCLR/src/coreclr) + +add_compile_options(-fexceptions) +add_definitions(-DUSE_STL) + +if(NOT WIN32) + include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/inc) + add_compile_options(-DPAL_STDCPP_COMPAT) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wno-null-arithmetic) + else(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wno-conversion-null -Wno-pointer-arith) + endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") +endif(NOT WIN32) + +if (APPLE) + include_directories(Pyjion/compat) + add_definitions(-DHOST_UNIX) + add_definitions(-DHOST_AMD64) + add_definitions(-DHOST_64BIT) + set(CLR_CMAKE_HOST_UNIX 1) +endif() + include_directories(CoreCLR/src/coreclr/src/inc) +#include(CoreCLR/eng/native/configurecompiler.cmake) + set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) add_library(pyjion SHARED ${SOURCES}) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare") - -option(WERROR "Turn warnings into errors" OFF) -if (WERROR) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") -endif () \ No newline at end of file +add_compile_options(-Wno-sign-compare) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 59e59c201..b10a1b129 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -23,13 +23,14 @@ * */ -#include "absint.h" -#include "taggedptr.h" #include #include #include #include +#include "absint.h" +#include "taggedptr.h" + #define NUM_ARGS(n) ((n)&0xFF) #define NUM_KW_ARGS(n) (((n)>>8) & 0xff) @@ -130,9 +131,9 @@ bool AbstractInterpreter::preprocess() { case SETUP_WITH: // not supported... return false; - case SETUP_LOOP: - blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), true)); - break; +// case SETUP_LOOP: +// blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), true)); +// break; case POP_BLOCK: { auto blockStart = blockStarts.back(); @@ -140,26 +141,26 @@ bool AbstractInterpreter::preprocess() { m_blockStarts[opcodeIndex] = blockStart.BlockStart; break; } - case SETUP_EXCEPT: - blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), false)); - ehKind.push_back(false); - break; +// case SETUP_EXCEPT: +// blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), false)); +// ehKind.push_back(false); +// break; case SETUP_FINALLY: blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), false)); ehKind.push_back(true); break; - case END_FINALLY: - m_endFinallyIsFinally[opcodeIndex] = ehKind.back(); - ehKind.pop_back(); - break; - case BREAK_LOOP: - for (auto iter = blockStarts.rbegin(); iter != blockStarts.rend(); ++iter) { - if (iter->IsLoop) { - m_breakTo[opcodeIndex] = *iter; - break; - } - } - break; +// case END_FINALLY: +// m_endFinallyIsFinally[opcodeIndex] = ehKind.back(); +// ehKind.pop_back(); +// break; +// case BREAK_LOOP: +// for (auto iter = blockStarts.rbegin(); iter != blockStarts.rend(); ++iter) { +// if (iter->IsLoop) { +// m_breakTo[opcodeIndex] = *iter; +// break; +// } +// } +// break; case LOAD_GLOBAL: { auto name = PyUnicode_AsUTF8(PyTuple_GetItem(m_code->co_names, oparg)); @@ -318,13 +319,11 @@ bool AbstractInterpreter::interpret() { case LOAD_CONST: { auto constSource = add_const_source(opcodeIndex, oparg); - - lastState.push( - AbstractValueWithSources( + auto value = AbstractValueWithSources( to_abstract(PyTuple_GetItem(m_code->co_consts, oparg)), constSource - ) - ); + ); + lastState.push(value); break; } case LOAD_FAST: @@ -394,8 +393,8 @@ bool AbstractInterpreter::interpret() { AbstractSource::combine(one.Sources, two.Sources), sources ); - - lastState.push(AbstractValueWithSources(binaryRes, sources)); + auto value = AbstractValueWithSources(binaryRes, sources); + lastState.push(value); } break; case POP_JUMP_IF_FALSE: @@ -593,7 +592,8 @@ bool AbstractInterpreter::interpret() { AbstractSource::combine(one.Sources, two.Sources), sources ); - lastState.push(AbstractValueWithSources(binaryRes, sources)); + auto value = AbstractValueWithSources(binaryRes, sources); + lastState.push(value); break; } } @@ -640,7 +640,7 @@ bool AbstractInterpreter::interpret() { // Pop the names tuple auto names = lastState.pop_no_escape(); - _ASSERTE(names.Value->kind() == AVK_Tuple); + assert(names.Value->kind() == AVK_Tuple); for (int i = 0; i < na; i++) { lastState.pop(); @@ -718,7 +718,8 @@ bool AbstractInterpreter::interpret() { if (opcode == UNARY_INVERT) one.escapes(); - lastState.push(AbstractValueWithSources(unaryRes, sources)); + auto value = AbstractValueWithSources(unaryRes, sources); + lastState.push(value); break; } case UNPACK_EX: @@ -793,8 +794,6 @@ bool AbstractInterpreter::interpret() { break; } - case SETUP_LOOP: - break; case POP_BLOCK: // Restore the stack state to what we had on entry lastState.m_stack = m_startStates[m_blockStarts[opcodeIndex]].m_stack; @@ -820,30 +819,30 @@ bool AbstractInterpreter::interpret() { lastState.pop(); lastState.pop(); break; - case CONTINUE_LOOP: - if (update_start_state(lastState, oparg)) { - queue.push_back(oparg); - } - // Done processing this basic block, we'll need to see a branch - // to the following opcodes before we'll process them. - goto next; - case BREAK_LOOP: - { - auto breakTo = m_breakTo[opcodeIndex]; - - // BREAK_LOOP does an unwind block, which restores the - // stack state to what it was when we entered the loop. So - // we get the start state for where the SETUP_LOOP happened - // here and propagate it to where we're breaking to. But we - // need to preserve our local state as that isn't restored. - auto startState = m_startStates[breakTo.BlockStart]; - startState.m_locals = lastState.m_locals; - if (update_start_state(startState, breakTo.BlockEnd)) { - queue.push_back(breakTo.BlockEnd); - } - - goto next; - } +// case CONTINUE_LOOP: +// if (update_start_state(lastState, oparg)) { +// queue.push_back(oparg); +// } +// // Done processing this basic block, we'll need to see a branch +// // to the following opcodes before we'll process them. +// goto next; +// case BREAK_LOOP: +// { +// auto breakTo = m_breakTo[opcodeIndex]; +// +// // BREAK_LOOP does an unwind block, which restores the +// // stack state to what it was when we entered the loop. So +// // we get the start state for where the SETUP_LOOP happened +// // here and propagate it to where we're breaking to. But we +// // need to preserve our local state as that isn't restored. +// auto startState = m_startStates[breakTo.BlockStart]; +// startState.m_locals = lastState.m_locals; +// if (update_start_state(startState, breakTo.BlockEnd)) { +// queue.push_back(breakTo.BlockEnd); +// } +// +// goto next; +// } case SETUP_FINALLY: { auto finallyState = lastState; @@ -855,34 +854,34 @@ bool AbstractInterpreter::interpret() { } } break; - case SETUP_EXCEPT: - { - auto ehState = lastState; - // Except is entered with the exception object, traceback, and exception - // type. TODO: We could type these stronger then they currently are typed - ehState.push(&Any); - ehState.push(&Any); - ehState.push(&Any); - if (update_start_state(ehState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { - queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); - } - } - break; - case END_FINALLY: - { - bool isFinally = m_endFinallyIsFinally[opcodeIndex]; - if (isFinally) { - // single value indicating whether we're unwinding or - // completed normally. - lastState.pop(); - } - else { - lastState.pop(); - lastState.pop(); - lastState.pop(); - } - break; - } +// case SETUP_EXCEPT: +// { +// auto ehState = lastState; +// // Except is entered with the exception object, traceback, and exception +// // type. TODO: We could type these stronger then they currently are typed +// ehState.push(&Any); +// ehState.push(&Any); +// ehState.push(&Any); +// if (update_start_state(ehState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { +// queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); +// } +// } +// break; +// case END_FINALLY: +// { +// bool isFinally = m_endFinallyIsFinally[opcodeIndex]; +// if (isFinally) { +// // single value indicating whether we're unwinding or +// // completed normally. +// lastState.pop(); +// } +// else { +// lastState.pop(); +// lastState.pop(); +// lastState.pop(); +// } +// break; +// } case FORMAT_VALUE: if ((oparg & FVS_MASK) == FVS_HAVE_SPEC) { // format spec @@ -941,7 +940,7 @@ bool AbstractInterpreter::merge_states(InterpreterState& newState, InterpreterSt if (mergeTo.m_locals != newState.m_locals) { // if (mergeTo.m_locals.get() != newState.m_locals.get()) { // need to merge locals... - _ASSERT(mergeTo.local_count() == newState.local_count()); + assert(mergeTo.local_count() == newState.local_count()); for (size_t i = 0; i < newState.local_count(); i++) { auto oldType = mergeTo.get_local(i); auto newType = oldType.merge_with(newState.get_local(i)); @@ -963,7 +962,7 @@ bool AbstractInterpreter::merge_states(InterpreterState& newState, InterpreterSt } else { // need to merge the stacks... - _ASSERT(mergeTo.stack_size() == newState.stack_size()); + assert(mergeTo.stack_size() == newState.stack_size()); for (size_t i = 0; i < newState.stack_size(); i++) { auto newType = mergeTo[i].merge_with(newState[i]); if (mergeTo[i] != newType) { @@ -1101,8 +1100,6 @@ void AbstractInterpreter::dump() { switch (opcode) { - case SETUP_LOOP: - case SETUP_EXCEPT: case SETUP_FINALLY: case JUMP_FORWARD: case FOR_ITER: @@ -1289,14 +1286,10 @@ char* AbstractInterpreter::opcode_name(int opcode) { OP_TO_STR(INPLACE_AND) OP_TO_STR(INPLACE_XOR) OP_TO_STR(INPLACE_OR) - OP_TO_STR(BREAK_LOOP) - OP_TO_STR(WITH_CLEANUP_START) - OP_TO_STR(WITH_CLEANUP_FINISH) OP_TO_STR(RETURN_VALUE) OP_TO_STR(IMPORT_STAR) OP_TO_STR(YIELD_VALUE) OP_TO_STR(POP_BLOCK) - OP_TO_STR(END_FINALLY) OP_TO_STR(POP_EXCEPT) OP_TO_STR(STORE_NAME) OP_TO_STR(DELETE_NAME) @@ -1324,9 +1317,6 @@ char* AbstractInterpreter::opcode_name(int opcode) { OP_TO_STR(POP_JUMP_IF_FALSE) OP_TO_STR(POP_JUMP_IF_TRUE) OP_TO_STR(LOAD_GLOBAL) - OP_TO_STR(CONTINUE_LOOP) - OP_TO_STR(SETUP_LOOP) - OP_TO_STR(SETUP_EXCEPT) OP_TO_STR(SETUP_FINALLY) OP_TO_STR(LOAD_FAST) OP_TO_STR(STORE_FAST) @@ -1620,7 +1610,7 @@ void AbstractInterpreter::extend_list_recursively(Local listTmp, size_t argCnt) } void AbstractInterpreter::extend_list(size_t argCnt) { - _ASSERTE(argCnt > 0); + assert(argCnt > 0); m_comp->emit_new_list(0); error_check("new list failed"); @@ -1713,7 +1703,7 @@ void AbstractInterpreter::extend_set_recursively(Local setTmp, size_t argCnt) { } void AbstractInterpreter::extend_set(size_t argCnt) { - _ASSERTE(argCnt > 0); + assert(argCnt > 0); m_comp->emit_new_set(); error_check("new set failed"); @@ -1765,7 +1755,7 @@ void AbstractInterpreter::extend_map_recursively(Local dictTmp, size_t argCnt) { } void AbstractInterpreter::extend_map(size_t argCnt) { - _ASSERTE(argCnt > 0); + assert(argCnt > 0); m_comp->emit_new_dict(0); error_check("new map failed"); @@ -1825,7 +1815,7 @@ void AbstractInterpreter::make_function(int oparg) { } void AbstractInterpreter::dec_stack(size_t size) { - _ASSERTE(m_stack.size() >= size); + assert(m_stack.size() >= size); for (size_t i = 0; i < size; i++) { m_stack.pop_back(); } @@ -1840,45 +1830,19 @@ void AbstractInterpreter::inc_stack(size_t size, bool kind) { // Frees our iteration temporary variable which gets allocated when we hit // a FOR_ITER. Used when we're breaking from the current loop. void AbstractInterpreter::free_iter_local() { - for (auto cur = m_blockStack.rbegin(); cur != m_blockStack.rend(); cur++) { - if ((*cur).Kind == SETUP_LOOP) { - if ((*cur).LoopVar.is_valid()) { - m_comp->emit_load_local((*cur).LoopVar); - m_comp->emit_pop_top(); - } - break; - } - } + } // Frees all of the iteration variables in a range. Used when we're // going to branch to a finally through multiple loops. void AbstractInterpreter::free_all_iter_locals(size_t to) { - for (size_t i = m_blockStack.size() - 1; i != to - 1; i--) { - if (m_blockStack[i].Kind == SETUP_LOOP) { - if (m_blockStack[i].LoopVar.is_valid()) { - m_comp->emit_load_local(m_blockStack[i].LoopVar); - m_comp->emit_pop_top(); - } - } - } + } // Frees all of our iteration variables. Used when we're unwinding the function // on an exception. void AbstractInterpreter::free_iter_locals_on_exception() { - int loopCount = 0; - for (auto cur = m_blockStack.rbegin(); cur != m_blockStack.rend(); cur++) { - if ((*cur).Kind == SETUP_LOOP) { - if ((*cur).LoopVar.is_valid()) { - m_comp->emit_load_local((*cur).LoopVar); - m_comp->emit_pop_top(); - } - } - else { - break; - } - } + } void AbstractInterpreter::periodic_work() { @@ -1998,7 +1962,7 @@ JittedCode* AbstractInterpreter::compile_worker() { } for (int curByte = 0; curByte < m_size; curByte += sizeof(_Py_CODEUNIT)) { - _ASSERTE(curByte % sizeof(_Py_CODEUNIT) == 0); + assert(curByte % sizeof(_Py_CODEUNIT) == 0); auto opcodeIndex = curByte; @@ -2085,83 +2049,6 @@ JittedCode* AbstractInterpreter::compile_worker() { m_comp->emit_dup_top_two(); break; case COMPARE_OP: compare_op(oparg, curByte, opcodeIndex); break; - case SETUP_LOOP: - // offset is relative to end of current instruction - m_blockStack.push_back( - BlockInfo(oparg + curByte + sizeof(_Py_CODEUNIT), SETUP_LOOP, m_blockStack.back().CurrentHandler) - ); - break; - case BREAK_LOOP: - case CONTINUE_LOOP: - // if we have finally blocks we need to unwind through them... - // used in exceptional case... - { - bool inFinally = false; - size_t loopIndex = -1, clearEh = -1; - for (size_t i = m_blockStack.size() - 1; i != -1; i--) { - if (m_blockStack[i].Kind == SETUP_LOOP) { - // we found our loop, we don't need additional processing... - loopIndex = i; - break; - } - else if (m_blockStack[i].Kind == END_FINALLY || m_blockStack[i].Kind == POP_EXCEPT) { - if (clearEh == -1) { - clearEh = i; - } - } - else if (m_blockStack[i].Kind == SETUP_FINALLY) { - // we need to run the finally before continuing to the loop... - // That means we need to spill the stack, branch to the finally, - // run it, and have the finally branch back to our oparg. - // CPython handles this by pushing the opcode to continue at onto - // the stack, and then pushing an integer value which indicates END_FINALLY - // should continue execution. Our END_FINALLY expects only a single value - // on the stack, and we also need to preserve any loop variables. - m_blockStack.data()[i].Flags |= byte == BREAK_LOOP ? EHF_BlockBreaks : EHF_BlockContinues; - - if (!inFinally) { - // only emit the branch to the first finally, subsequent branches - // to other finallys will be handled by the END_FINALLY code. But we - // need to mark those finallys as needing special handling. - inFinally = true; - if (clearEh != -1) { - unwind_eh( - m_blockStack[clearEh].CurrentHandler, - m_allHandlers[m_blockStack[i].CurrentHandler].BackHandler - ); - } - m_comp->emit_int(byte == BREAK_LOOP ? EHF_BlockBreaks : EHF_BlockContinues); - m_comp->emit_branch(BranchAlways, m_allHandlers[m_blockStack[i].CurrentHandler].ErrorTarget); - if (byte == CONTINUE_LOOP) { - m_blockStack.data()[i].ContinueOffset = oparg; - } - } - } - } - - if (!inFinally) { - if (clearEh != -1) { - unwind_eh( - m_blockStack[clearEh].CurrentHandler, - m_blockStack[loopIndex].CurrentHandler - ); - } - if (byte != CONTINUE_LOOP) { - free_iter_local(); - } - - if (byte == BREAK_LOOP) { - assert(loopIndex != -1); - m_comp->emit_branch(BranchLeave, getOffsetLabel(m_blockStack[loopIndex].EndOffset)); - } - else { - periodic_work(); - m_comp->emit_branch(BranchLeave, getOffsetLabel(oparg)); - } - } - - } - break; case LOAD_BUILD_CLASS: m_comp->emit_load_build_class(); error_check("load build class failed"); @@ -2366,14 +2253,14 @@ JittedCode* AbstractInterpreter::compile_worker() { // Currently we only optimize floating point numbers.. if (one.Value->kind() == AVK_Integer && two.Value->kind() == AVK_Float) { // tagged ints might be objects, so we track the stack kind as object - _ASSERTE(m_stack[m_stack.size() - 1] == STACK_KIND_OBJECT); - _ASSERTE(m_stack[m_stack.size() - 2] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 1] == STACK_KIND_OBJECT); + assert(m_stack[m_stack.size() - 2] == STACK_KIND_VALUE); if (byte == BINARY_AND || byte == INPLACE_AND || byte == INPLACE_OR || byte == BINARY_OR || byte == INPLACE_LSHIFT || byte == BINARY_LSHIFT || byte == INPLACE_RSHIFT || byte == BINARY_RSHIFT || byte == INPLACE_XOR || byte == BINARY_XOR) { char buf[100]; - sprintf_s(buf, "unsupported operand type(s) for %s: 'float' and 'int'", op_to_string(byte)); + sprintf(buf, "unsupported operand type(s) for %s: 'float' and 'int'", op_to_string(byte)); m_comp->emit_pyerr_setstring(PyExc_TypeError, buf); branch_raise(); break; @@ -2409,8 +2296,8 @@ JittedCode* AbstractInterpreter::compile_worker() { inc_stack(1, STACK_KIND_VALUE); break; } else if (one.Value->kind() == AVK_Float && two.Value->kind() == AVK_Float) { - _ASSERTE(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); - _ASSERTE(m_stack[m_stack.size() - 2] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 2] == STACK_KIND_VALUE); dec_stack(2); @@ -2514,17 +2401,6 @@ JittedCode* AbstractInterpreter::compile_worker() { case FOR_ITER: { BlockInfo *loopBlock = nullptr; - for (size_t blockIndex = m_blockStack.size() - 1; blockIndex != -1; blockIndex--) { - if (m_blockStack[blockIndex].Kind == SETUP_LOOP) { - // save our iter variable so we can free it on break, continue, return, and - // when encountering an exception. - loopBlock = &m_blockStack.data()[blockIndex]; - //loopOpt1 = m_blockStack.data()[blockIndex].LoopOpt1; - //loopOpt2 = m_blockStack.data()[blockIndex].LoopOpt2; - //inLoop = true; - break; - } - } for_iter( curByte + oparg + sizeof(_Py_CODEUNIT), opcodeIndex, @@ -2620,32 +2496,6 @@ JittedCode* AbstractInterpreter::compile_worker() { break; } break; - case SETUP_EXCEPT: - { - auto handlerLabel = getOffsetLabel(oparg + curByte + sizeof(_Py_CODEUNIT)); - - auto blockInfo = BlockInfo(oparg + curByte + sizeof(_Py_CODEUNIT), SETUP_EXCEPT, m_allHandlers.size()); - m_blockStack.push_back(blockInfo); - - m_allHandlers.push_back( - ExceptionHandler( - m_allHandlers.size(), - ExceptionVars(m_comp), - m_comp->emit_define_label(), - m_comp->emit_define_label(), - handlerLabel, - m_stack, - EHF_TryExcept - ) - ); - - vector newStack = m_stack; - for (int j = 0; j < 3; j++) { - newStack.push_back(STACK_KIND_OBJECT); - } - m_offsetStack[oparg + curByte + sizeof(_Py_CODEUNIT)] = newStack; - } - break; case SETUP_FINALLY: { auto handlerLabel = getOffsetLabel(oparg + curByte + sizeof(_Py_CODEUNIT)); @@ -2671,124 +2521,6 @@ JittedCode* AbstractInterpreter::compile_worker() { break; case POP_EXCEPT: pop_except(); break; case POP_BLOCK: compile_pop_block(); break; - case END_FINALLY: - { - // CPython excepts END_FINALLY can be entered in 1 of 3 ways: - // 1) With a status code for why the finally is unwinding, indicating a RETURN - // or a continue. In this case there is an extra retval on the stack - // 2) With an excpetion class which is being raised. In this case there are 2 extra - // values on the stack, the exception value, and the traceback. - // 3) After the try block has completed normally. In this case None is on the stack. - // - // That means in CPython this opcode can be branched to with 1 of 3 different stack - // depths, and the CLR doesn't like that. Worse still the rest of the generated - // byte code assumes this is true. For case 2 an except handler includes tests - // and pops which remove the 3 values from the class. For case 3 the byte code - // at the end of the finally range includes the opcode to load None. - // - // END_FINALLY can also be encountered w/o a SETUP_FINALLY, as happens when it's used - // solely for re-throwing exceptions. - - auto curBlock = m_blockStack.back(); - auto ehInfo = m_allHandlers[curBlock.CurrentHandler]; - m_blockStack.pop_back(); - - if (m_endFinallyIsFinally[opcodeIndex]) { - int flags = curBlock.Flags; - - // We're actually ending a finally. If we're in an exceptional case we - // need to re-throw, otherwise we need to just continue execution. Our - // exception handling code will only push the exception type on in this case. - auto noException = m_comp->emit_define_label(); - dec_stack(); - m_comp->emit_store_local(ehInfo.ExVars.FinallyExc); - m_comp->emit_load_local(ehInfo.ExVars.FinallyExc); - m_comp->emit_ptr(Py_None); - m_comp->emit_dup(); - m_comp->emit_incref(); - m_comp->emit_branch(BranchEqual, noException); - - if (flags & EHF_BlockBreaks) { - unwind_loop(ehInfo.ExVars.FinallyExc, EHF_BlockBreaks, 0); - } - - if (flags & EHF_BlockContinues) { - unwind_loop(ehInfo.ExVars.FinallyExc, EHF_BlockContinues, curBlock.ContinueOffset); - } - - if (flags & EHF_BlockReturns) { - auto exceptional = m_comp->emit_define_label(); - m_comp->emit_load_local(ehInfo.ExVars.FinallyExc); - m_comp->emit_int(EHF_BlockReturns); - m_comp->emit_compare_equal(); - m_comp->emit_branch(BranchFalse, exceptional); - - bool hasOuterFinally = false; - size_t clearEh = -1; - for (size_t i = m_blockStack.size() - 1; i != -1; i--) { - if (m_blockStack[i].Kind == SETUP_FINALLY) { - // need to dispatch to outer finally... - m_comp->emit_load_local(ehInfo.ExVars.FinallyExc); - m_comp->emit_branch(BranchAlways, m_allHandlers[m_blockStack[i].CurrentHandler].ErrorTarget); - hasOuterFinally = true; - break; - } - else if (m_blockStack[i].Kind == POP_EXCEPT || m_blockStack[i].Kind == END_FINALLY) { - if (clearEh == -1) { - clearEh = i; - } - } - - } - - if (clearEh != -1) { - unwind_eh(m_blockStack[clearEh].CurrentHandler); - } - - if (!hasOuterFinally) { - m_comp->emit_branch(BranchLeave, m_retLabel); - } - - m_comp->emit_mark_label(exceptional); - } - - // re-raise the exception... - free_iter_locals_on_exception(); - - m_comp->emit_load_local(ehInfo.ExVars.FinallyTb); - m_comp->emit_load_local(ehInfo.ExVars.FinallyValue); - m_comp->emit_load_local(ehInfo.ExVars.FinallyExc); - m_comp->emit_restore_err(); - unwind_eh(curBlock.CurrentHandler, m_blockStack.back().CurrentHandler); - - auto ehBlock = get_ehblock(); - - clean_stack_for_reraise(); - m_comp->emit_branch(BranchAlways, ehBlock.ReRaise); - - m_comp->emit_mark_label(noException); - } - else { - // END_FINALLY is marking the EH rethrow. The byte code branches - // around this in the non-exceptional case. - - // If we haven't sent a branch to this END_FINALLY then we have - // a bare try/except: which handles all exceptions. In that case - // we have no values to pop off, and this code will never be invoked - // anyway. - if (m_offsetStack.find(curByte) != m_offsetStack.end()) { - dec_stack(3); - free_iter_locals_on_exception(); - m_comp->emit_restore_err(); - - unwind_eh(curBlock.CurrentHandler, m_blockStack.back().CurrentHandler); - clean_stack_for_reraise(); - - m_comp->emit_branch(BranchAlways, get_ehblock().ReRaise); - } - } - } - break; case YIELD_FROM: case YIELD_VALUE: @@ -2811,9 +2543,6 @@ JittedCode* AbstractInterpreter::compile_worker() { int_error_check("import star failed"); break; case SETUP_WITH: - case WITH_CLEANUP_START: - case WITH_CLEANUP_FINISH: - return nullptr; case BUILD_MAP_UNPACK_WITH_CALL: /* TODO: Finish implementation @@ -3008,7 +2737,7 @@ JittedCode* AbstractInterpreter::compile_worker() { void AbstractInterpreter::compile_pop_block() { auto curHandler = m_blockStack.back(); m_blockStack.pop_back(); - if (curHandler.Kind == SETUP_FINALLY || curHandler.Kind == SETUP_EXCEPT) { + if (curHandler.Kind == SETUP_FINALLY) { // convert block into an END_FINALLY/POP_EXCEPT BlockInfo auto back = m_blockStack.back(); @@ -3017,7 +2746,7 @@ void AbstractInterpreter::compile_pop_block() { auto newBlock = BlockInfo( back.EndOffset, - curHandler.Kind == SETUP_FINALLY ? END_FINALLY : POP_EXCEPT, + curHandler.Kind == SETUP_FINALLY ? NULL : POP_EXCEPT, // TODO : Find equivalent m_allHandlers.size(), curHandler.Flags, curHandler.ContinueOffset @@ -3142,27 +2871,7 @@ void AbstractInterpreter::emit_raise_and_free(size_t handlerIndex) { void AbstractInterpreter::unwind_loop(Local finallyReason, EhFlags branchKind, int continueOffset) { size_t clearEh = -1; for (size_t i = m_blockStack.size() - 1; i != -1; i--) { - if (m_blockStack[i].Kind == SETUP_LOOP) { - if (clearEh != -1) { - unwind_eh(m_blockStack[clearEh].CurrentHandler, m_blockStack[i].CurrentHandler); - } - - // We need to emit a BranchLeave here in case we're inside of a nested finally block. Finally - // blocks leave a value on the stack during the entire computation, so we need to clear the - // stack before branching away. Right now we just always emit this BranchNotEqual/BranchLeave pattern - if (branchKind == EHF_BlockContinues) { - periodic_work(); - } - auto target = getOffsetLabel(branchKind == EHF_BlockContinues ? continueOffset : m_blockStack[i].EndOffset); - auto noBranch = m_comp->emit_define_label(); - m_comp->emit_load_local(finallyReason); - m_comp->emit_int(branchKind); - m_comp->emit_branch(BranchNotEqual, noBranch); - m_comp->emit_branch(BranchLeave, target); - m_comp->emit_mark_label(noBranch); - break; - } - else if (m_blockStack[i].Kind == SETUP_FINALLY) { + if (m_blockStack[i].Kind == SETUP_FINALLY) { // need to dispatch to outer finally... if (clearEh != -1) { unwind_eh( @@ -3175,7 +2884,7 @@ void AbstractInterpreter::unwind_loop(Local finallyReason, EhFlags branchKind, i m_comp->emit_branch(BranchAlways, m_allHandlers[m_blockStack[i].CurrentHandler].ErrorTarget); break; } - else if (m_blockStack[i].Kind == POP_EXCEPT || m_blockStack[i].Kind == END_FINALLY) { + else if (m_blockStack[i].Kind == POP_EXCEPT) { if (clearEh == -1) { clearEh = i; } @@ -3417,7 +3126,6 @@ JittedCode* AbstractInterpreter::compile() { bool AbstractInterpreter::can_skip_lasti_update(int opcodeIndex) { switch (GET_OPCODE(opcodeIndex)) { case DUP_TOP: - case SETUP_EXCEPT: case NOP: case ROT_TWO: case ROT_THREE: @@ -3426,9 +3134,6 @@ bool AbstractInterpreter::can_skip_lasti_update(int opcodeIndex) { case POP_JUMP_IF_TRUE: case POP_TOP: case DUP_TOP_TWO: - case BREAK_LOOP: - case CONTINUE_LOOP: - case END_FINALLY: case LOAD_CONST: case JUMP_FORWARD: case JUMP_ABSOLUTE: @@ -3445,7 +3150,7 @@ void AbstractInterpreter::store_fast(int local, int opcodeIndex) { // We only optimize floats so far... if (stackValue.Value->kind() == AVK_Float) { - _ASSERTE(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); m_comp->emit_store_local(get_optimized_local(local, AVK_Float)); dec_stack(); return; @@ -3457,7 +3162,7 @@ void AbstractInterpreter::store_fast(int local, int opcodeIndex) { } } - _ASSERTE(m_stack[m_stack.size() - 1] == STACK_KIND_OBJECT); + assert(m_stack[m_stack.size() - 1] == STACK_KIND_OBJECT); m_comp->emit_store_fast(local); dec_stack(); } @@ -3492,7 +3197,7 @@ void AbstractInterpreter::return_value(int opcodeIndex) { auto stackInfo = get_stack_info(opcodeIndex); // We only optimize floats so far... if (stackInfo[stackInfo.size() - 1].Value->kind() == AVK_Float) { - _ASSERTE(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); // we need to convert the returned floating point value back into a boxed float. m_comp->emit_box_float(); @@ -3529,23 +3234,10 @@ void AbstractInterpreter::return_value(int opcodeIndex) { m_comp->emit_branch(BranchAlways, m_allHandlers[m_blockStack[blockIndex].CurrentHandler].ErrorTarget); } } - else if (m_blockStack[blockIndex].Kind == POP_EXCEPT || m_blockStack[blockIndex].Kind == END_FINALLY) { + else if (m_blockStack[blockIndex].Kind == POP_EXCEPT) { // we need to restore the previous exception before we return if (clearEh == -1) { clearEh = blockIndex; - if (m_blockStack[blockIndex].Kind == END_FINALLY) { - m_comp->emit_load_local(m_allHandlers[m_blockStack[blockIndex].CurrentHandler].ExVars.FinallyTb); - m_comp->emit_pop_top(); - m_comp->emit_load_local(m_allHandlers[m_blockStack[blockIndex].CurrentHandler].ExVars.FinallyValue); - m_comp->emit_pop_top(); - m_comp->emit_load_local(m_allHandlers[m_blockStack[blockIndex].CurrentHandler].ExVars.FinallyExc); - m_comp->emit_pop_top(); - - m_comp->emit_null(); - m_comp->emit_null(); - m_comp->emit_null(); - m_comp->emit_restore_err(); - } } } } @@ -3691,8 +3383,8 @@ void AbstractInterpreter::compare_op(int compareType, int& i, int opcodeIndex) { if (stackInfo[stackInfo.size() - 1].Value->kind() == AVK_Float && stackInfo[stackInfo.size() - 2].Value->kind() == AVK_Float) { - _ASSERTE(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); - _ASSERTE(m_stack[m_stack.size() - 2] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 1] == STACK_KIND_VALUE); + assert(m_stack[m_stack.size() - 2] == STACK_KIND_VALUE); m_comp->emit_compare_float(compareType); dec_stack(); @@ -3925,7 +3617,7 @@ void AbstractInterpreter::debug_log(const char* fmt, ...) { va_start(args, fmt); const int bufferSize = 181; char* buffer = new char[bufferSize]; - vsprintf_s(buffer, bufferSize, fmt, args); + vsnprintf(buffer, bufferSize, fmt, args); va_end(args); m_comp->emit_debug_msg(buffer); } diff --git a/Pyjion/absint.h b/Pyjion/absint.h index 1f2dcddd7..5a2049bac 100644 --- a/Pyjion/absint.h +++ b/Pyjion/absint.h @@ -26,12 +26,6 @@ #ifndef ABSINT_H #define ABSINT_H -#include - -#ifndef _WIN32 -#include -#endif - #include #include #include @@ -421,8 +415,8 @@ struct AbstractLocalInfo { } AbstractLocalInfo(AbstractValueWithSources valueInfo, bool isUndefined = false) { - _ASSERTE(valueInfo.Value != nullptr); - _ASSERTE(!(valueInfo.Value == &Undefined && !isUndefined)); + assert(valueInfo.Value != nullptr); + assert(!(valueInfo.Value == &Undefined && !isUndefined)); ValueInfo = valueInfo; IsMaybeUndefined = isUndefined; } diff --git a/Pyjion/absvalue.h b/Pyjion/absvalue.h index a3646a5ed..da8e022ca 100644 --- a/Pyjion/absvalue.h +++ b/Pyjion/absvalue.h @@ -28,6 +28,7 @@ #include #include + #include "cowvector.h" class AbstractValue; diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 827fe99e6..efbd22a5e 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -27,10 +27,9 @@ #define CEE_H #define FEATURE_NO_HOST -#define USE_STL + #include #include -#include #include #include #include @@ -44,8 +43,11 @@ #include #include -#include "corjit.h" -#include "utilcode.h" +#include +#include +#include + + #include "openum.h" using namespace std; diff --git a/Pyjion/codemodel.h b/Pyjion/codemodel.h index 7a919c395..b5271a7ba 100644 --- a/Pyjion/codemodel.h +++ b/Pyjion/codemodel.h @@ -27,7 +27,7 @@ #define CODEMODEL_H #define FEATURE_NO_HOST -#define USE_STL + #include #include #include diff --git a/Pyjion/compat/malloc.h b/Pyjion/compat/malloc.h new file mode 100644 index 000000000..bf845a528 --- /dev/null +++ b/Pyjion/compat/malloc.h @@ -0,0 +1 @@ +// Dummy file.. \ No newline at end of file diff --git a/Pyjion/intrins.cpp b/Pyjion/intrins.cpp index c2c74b50d..4446cc1fd 100644 --- a/Pyjion/intrins.cpp +++ b/Pyjion/intrins.cpp @@ -25,9 +25,10 @@ * Portions lifted from CPython under the PSF license. */ +#include + #include "intrins.h" #include "taggedptr.h" -#include #ifdef _MSC_VER diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index dc1bb6170..64233f1ec 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -30,7 +30,6 @@ #define FEATURE_NO_HOST #include -//#include #include #include #include diff --git a/Pyjion/jitinit.h b/Pyjion/jitinit.h index 92ff314d6..7d062c3cd 100644 --- a/Pyjion/jitinit.h +++ b/Pyjion/jitinit.h @@ -3,7 +3,7 @@ #define FEATURE_NO_HOST -#define USE_STL + #include #include #include diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index b9c34df1a..88a777166 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -23,14 +23,14 @@ * */ -#include -#define PAL_STDCPP_COMPAT 1 -#include "pycomp.h" +#include #include #include #include +#include "pycomp.h" + HRESULT __stdcall GetCORSystemDirectoryInternal(SString& pbuffer) { printf("get cor system\n"); return S_OK; From a0eed614abbb638bc3e6bc855ebf6a91ca807274 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 15:45:24 +1000 Subject: [PATCH 013/273] More portability updates --- BuildDebugPython.sh | 4 ++-- BuildDeps.cmd | 4 ++-- CMakeLists.txt | 5 +++-- Pyjion/absint.cpp | 1 + Pyjion/cee.h | 2 -- Pyjion/jitinit.h | 2 -- Pyjion/pycomp.cpp | 3 +-- 7 files changed, 9 insertions(+), 12 deletions(-) diff --git a/BuildDebugPython.sh b/BuildDebugPython.sh index d7f56fac3..df3d089bc 100644 --- a/BuildDebugPython.sh +++ b/BuildDebugPython.sh @@ -1,5 +1,5 @@ echo Building CPython in debug mode for x64 ... -cd Python/ +pushd Python/ ./configure --with-pydebug make -s -j2 -cd .. \ No newline at end of file +popd \ No newline at end of file diff --git a/BuildDeps.cmd b/BuildDeps.cmd index 879e73eee..e657a275d 100644 --- a/BuildDeps.cmd +++ b/BuildDeps.cmd @@ -19,7 +19,7 @@ goto Usage :BuildCoreCLR pushd coreclr -call build.cmd %__BuildArch% %__BuildType% skipmscorlib skiptests +call build.cmd %__BuildArch% %__BuildType% IF ERRORLEVEL 1 goto Error mkdir ..\Libs\%__BuildType%\%__BuildArch%\ @@ -41,7 +41,7 @@ if /i "%__BuildArch%" == "x64" set arch=amd64 set SUFFIX= if /i "%__BuildType%" == "Debug" set SUFFIX=_d -copy %arch%\python35%SUFFIX%.lib ..\..\Libs\%__BuildType%\%__BuildArch%\ +copy %arch%\python39%SUFFIX%.lib ..\..\Libs\%__BuildType%\%__BuildArch%\ popd :Done diff --git a/CMakeLists.txt b/CMakeLists.txt index 59a1638e2..2591bea7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,8 +18,9 @@ add_compile_options(-fexceptions) add_definitions(-DUSE_STL) if(NOT WIN32) - include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/inc) + include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) + add_compile_options(-fdeclspec) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-null-arithmetic) else(CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -36,8 +37,8 @@ if (APPLE) endif() include_directories(CoreCLR/src/coreclr/src/inc) +include_directories(CoreCLR/artifacts/bin/coreclr/OSX.x64.Debug/inc) -#include(CoreCLR/eng/native/configurecompiler.cmake) set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) add_library(pyjion SHARED ${SOURCES}) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index b10a1b129..01e403a01 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -23,6 +23,7 @@ * */ +#include #include #include #include diff --git a/Pyjion/cee.h b/Pyjion/cee.h index efbd22a5e..8892314c7 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -45,8 +45,6 @@ #include #include -#include - #include "openum.h" diff --git a/Pyjion/jitinit.h b/Pyjion/jitinit.h index 7d062c3cd..85b867951 100644 --- a/Pyjion/jitinit.h +++ b/Pyjion/jitinit.h @@ -20,10 +20,8 @@ #include #include -#include #include -CorJitInfo g_corJitInfo; ICorJitCompiler* g_jit; extern "C" __declspec(dllexport) void JitInit() diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 88a777166..27f133aa4 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -24,8 +24,7 @@ */ #include - -#include +#include #include #include From 3dca58b40c66de275eba6518ee665285d71af76a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 19:04:48 +1000 Subject: [PATCH 014/273] Remove IExecutionEngine implementation per removal in coreCLR see https://github.com/dotnet/runtime/pull/33640 --- Pyjion/cee.h | 309 --------------------------------------------------- 1 file changed, 309 deletions(-) diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 8892314c7..58300519b 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -50,315 +50,6 @@ using namespace std; -class CExecutionEngine : public IExecutionEngine, public IEEMemoryManager { -public: - HANDLE m_codeHeap, m_heap; - DWORD m_tlsIndex; - PTLS_CALLBACK_FUNCTION* m_callbacks; - - - CExecutionEngine() { - m_codeHeap = HeapCreate(HEAP_CREATE_ENABLE_EXECUTE, 0, 0); - m_heap = HeapCreate(0, 0, 0); - m_tlsIndex = TlsAlloc(); - // We can't use new[] here because utilcode isn't spun up yet... - m_callbacks = (PTLS_CALLBACK_FUNCTION*)::HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(PTLS_CALLBACK_FUNCTION) * MAX_PREDEFINED_TLS_SLOT); - } - - ~CExecutionEngine() { - ::HeapDestroy(m_codeHeap); - ::HeapDestroy(m_heap); - TlsFree(m_tlsIndex); - ::HeapFree(GetProcessHeap(), 0, m_callbacks); - } - - // Thread Local Storage is based on logical threads. The underlying - // implementation could be threads, fibers, or something more exotic. - // Slot numbers are predefined. This is not a general extensibility - // mechanism. - - // Associate a callback function for releasing TLS on thread/fiber death. - // This can be NULL. - void TLS_AssociateCallback(DWORD slot, PTLS_CALLBACK_FUNCTION callback) { - m_callbacks[slot] = callback; - } - - // Get the TLS block for fast Get/Set operations - PVOID* TLS_GetDataBlock() { - //printf("get data block\r\n"); - PVOID* block = (PVOID*)TlsGetValue(m_tlsIndex); - if (block == nullptr) { - block = new PVOID[MAX_PREDEFINED_TLS_SLOT]; - memset(block, 0, sizeof(PVOID) * MAX_PREDEFINED_TLS_SLOT); - if (block != nullptr) { - TlsSetValue(m_tlsIndex, block); - } - } - return block; - } - - // Get the value at a slot - PVOID TLS_GetValue(DWORD slot) { - auto block = TLS_GetDataBlock(); - if (block != nullptr) { - return block[slot]; - } - return nullptr; - } - - // Get the value at a slot, return FALSE if TLS info block doesn't exist - BOOL TLS_CheckValue(DWORD slot, PVOID * pValue) { - auto block = TLS_GetDataBlock(); - if (block != nullptr) { - *pValue = block[slot]; - return TRUE; - } - return FALSE; - } - - // Set the value at a slot - void TLS_SetValue(DWORD slot, PVOID pData) { - auto block = TLS_GetDataBlock(); - if (block != nullptr) { - block[slot] = pData; - } - } - - // Free TLS memory block and make callback - void TLS_ThreadDetaching() { - auto block = TLS_GetDataBlock(); - if (block != nullptr) { - for (int i = 0; i < MAX_PREDEFINED_TLS_SLOT; i++) { - if (m_callbacks[i] != nullptr) { - m_callbacks[i](block[i]); - } - } - delete[] block; - } - } - - // Critical Sections are sometimes exposed to the host and therefore need to be - // reflected from all CLR DLLs to the EE. - // - // In addition, we always monitor interactions between the lock & the GC, based - // on the GC mode in which the lock is acquired and we restrict what operations - // are permitted while holding the lock based on this. - // - // Finally, we we rank all our locks to prevent deadlock across all the DLLs of - // the CLR. This is the level argument to CreateLock. - // - // All usage of these locks must be exception-safe. To achieve this, we suggest - // using Holders (see holder.h & crst.h). In fact, within the EE code cannot - // hold locks except by using exception-safe holders. - - CRITSEC_COOKIE CreateLock(LPCSTR szTag, LPCSTR level, CrstFlags flags) { - LPCRITICAL_SECTION critSec = new CRITICAL_SECTION(); - InitializeCriticalSection(critSec); - return (CRITSEC_COOKIE)critSec; - } - - void DestroyLock(CRITSEC_COOKIE lock) { - DeleteCriticalSection((LPCRITICAL_SECTION)lock); - delete (CRITICAL_SECTION*)lock; - } - - void AcquireLock(CRITSEC_COOKIE lock) { - EnterCriticalSection((LPCRITICAL_SECTION)lock); - } - - void ReleaseLock(CRITSEC_COOKIE lock) { - LeaveCriticalSection((LPCRITICAL_SECTION)lock); - } - - EVENT_COOKIE CreateAutoEvent(BOOL bInitialState) { - return (EVENT_COOKIE)::CreateEventW(NULL, FALSE, bInitialState, NULL); - } - - EVENT_COOKIE CreateManualEvent(BOOL bInitialState) { - return (EVENT_COOKIE)::CreateEventW(NULL, TRUE, bInitialState, NULL); - } - - void CloseEvent(EVENT_COOKIE event) { - CloseHandle((HANDLE)event); - } - - BOOL ClrSetEvent(EVENT_COOKIE event) { - return SetEvent((HANDLE)event); - } - - BOOL ClrResetEvent(EVENT_COOKIE event) { - return ResetEvent((HANDLE)event); - } - - DWORD WaitForEvent(EVENT_COOKIE event, DWORD dwMilliseconds, BOOL bAlertable) { - return ::WaitForSingleObjectEx((HANDLE)event, dwMilliseconds, bAlertable); - } - - DWORD WaitForSingleObject(HANDLE handle, DWORD dwMilliseconds) { - return ::WaitForSingleObject(handle, dwMilliseconds); - } - - // OS header file defines CreateSemaphore. - SEMAPHORE_COOKIE ClrCreateSemaphore(DWORD dwInitial, DWORD dwMax) { - return (SEMAPHORE_COOKIE)CreateSemaphoreW(NULL, dwInitial, dwMax, NULL); - } - - void ClrCloseSemaphore(SEMAPHORE_COOKIE semaphore) { - CloseHandle((HANDLE)semaphore); - } - - DWORD ClrWaitForSemaphore(SEMAPHORE_COOKIE semaphore, DWORD dwMilliseconds, BOOL bAlertable) { - return ::WaitForSingleObjectEx((HANDLE)semaphore, dwMilliseconds, bAlertable); - } - - BOOL ClrReleaseSemaphore(SEMAPHORE_COOKIE semaphore, LONG lReleaseCount, LONG *lpPreviousCount) { - return ::ReleaseSemaphore(semaphore, lReleaseCount, lpPreviousCount); - } - - MUTEX_COOKIE ClrCreateMutex(LPSECURITY_ATTRIBUTES lpMutexAttributes, BOOL bInitialOwner, LPCTSTR lpName) { - printf("create mutex not impl\r\n"); - return NULL; - } - - DWORD ClrWaitForMutex(MUTEX_COOKIE mutex, DWORD dwMilliseconds, BOOL bAlertable) { - printf("wait for mutex impl\r\n"); - return 0; - } - - BOOL ClrReleaseMutex(MUTEX_COOKIE mutex) { - printf("release mutex\r\n"); - return FALSE; - } - - void ClrCloseMutex(MUTEX_COOKIE mutex) { - printf("close mutex\r\n"); - } - - DWORD ClrSleepEx(DWORD dwMilliseconds, BOOL bAlertable) { - return ::SleepEx(dwMilliseconds, bAlertable); - } - - BOOL ClrAllocationDisallowed() { - return FALSE; - } - - void GetLastThrownObjectExceptionFromThread(void **ppvException) { - ppvException = nullptr; - } - - ULONG AddRef(void) { - return InterlockedIncrement(&m_refCount); - } - - HRESULT QueryInterface(REFIID iid, void** ppvObject) { - if (iid == IID_IUnknown) { - *ppvObject = (void*)static_cast(this); - return S_OK; - } - else if (iid == IID_IExecutionEngine) { - *ppvObject = (void*)static_cast(this); - return S_OK; - } - else if (iid == IID_IEEMemoryManager) { - *ppvObject = (void*)static_cast(this); - return S_OK; - } - return E_NOINTERFACE; - } - - ULONG Release(void) { - return InterlockedDecrement(&m_refCount); - } - - LPVOID ClrVirtualAlloc( - LPVOID lpAddress, // region to reserve or commit - SIZE_T dwSize, // size of region - DWORD flAllocationType, // type of allocation - DWORD flProtect // type of access protection - ) { - return ::VirtualAlloc(lpAddress, dwSize, flAllocationType, flProtect); - } - - BOOL ClrVirtualFree( - LPVOID lpAddress, // address of region - SIZE_T dwSize, // size of region - DWORD dwFreeType // operation type - ) { - return ::VirtualFree(lpAddress, dwSize, dwFreeType); - } - - SIZE_T ClrVirtualQuery( - const void* lpAddress, // address of region - PMEMORY_BASIC_INFORMATION lpBuffer, // information buffer - SIZE_T dwLength // size of buffer - ) { - return ::VirtualQuery(lpAddress, lpBuffer, dwLength); - } - - BOOL ClrVirtualProtect( - LPVOID lpAddress, // region of committed pages - SIZE_T dwSize, // size of the region - DWORD flNewProtect, // desired access protection - DWORD* lpflOldProtect // old protection - ) { - return ::VirtualProtect(lpAddress, dwSize, flNewProtect, lpflOldProtect); - } - - HANDLE ClrGetProcessHeap() { - return m_heap; - } - - HANDLE ClrHeapCreate( - DWORD flOptions, // heap allocation attributes - SIZE_T dwInitialSize, // initial heap size - SIZE_T dwMaximumSize // maximum heap size - ) { - return ::HeapCreate(flOptions, dwInitialSize, dwMaximumSize); - } - - BOOL ClrHeapDestroy( - HANDLE hHeap // handle to heap - ) { - return ::HeapDestroy(hHeap); - } - - LPVOID ClrHeapAlloc( - HANDLE hHeap, // handle to private heap block - DWORD dwFlags, // heap allocation control - SIZE_T dwBytes // number of bytes to allocate - ) { - return ::HeapAlloc(hHeap, dwFlags, dwBytes); - } - - BOOL ClrHeapFree( - HANDLE hHeap, // handle to heap - DWORD dwFlags, // heap free options - LPVOID lpMem // pointer to memory - ) { - return ::HeapFree(hHeap, dwFlags, lpMem); - } - - BOOL ClrHeapValidate( - HANDLE hHeap, // handle to heap - DWORD dwFlags, // heap access options - const void* lpMem // optional pointer to memory block - ) { - return ::HeapValidate(hHeap, dwFlags, lpMem); - } - - HANDLE ClrGetProcessExecutableHeap() { - if (m_executableHeap == NULL) { - m_executableHeap = HeapCreate(HEAP_CREATE_ENABLE_EXECUTE, 0, 0); - } - return m_executableHeap; - } - - -private: - ULONG m_refCount; - HANDLE m_executableHeap; -}; // interface IExecutionEngine - class CCorJitHost : public ICorJitHost { void * allocateMemory(size_t size, bool usePageAllocator = false) { From 60c03409e8b9eecafec4aaac8e4e8aebb6ab698f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 28 Sep 2020 19:16:45 +1000 Subject: [PATCH 015/273] Remove a bunch of stuff that doesn't exist in the JIT anymore --- Pyjion/jitinfo.h | 177 +---------------------------------------------- 1 file changed, 3 insertions(+), 174 deletions(-) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 64233f1ec..a0fd17846 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -50,7 +50,6 @@ using namespace std; class CorJitInfo : public ICorJitInfo, public JittedCode { - CExecutionEngine& m_executionEngine; void* m_codeAddr; void* m_dataAddr; PyCodeObject *m_code; @@ -58,7 +57,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { public: - CorJitInfo(CExecutionEngine& executionEngine, PyCodeObject* code, UserModule* module) : m_executionEngine(executionEngine) { + CorJitInfo(PyCodeObject* code, UserModule* module) { m_codeAddr = m_dataAddr = nullptr; m_code = code; m_module = module; @@ -78,13 +77,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return m_codeAddr; } - /* ICorJitInfo */ - IEEMemoryManager* getMemoryManager() { - return &m_executionEngine; - } void freeMem(PVOID code) { - HeapFree(m_executionEngine.m_codeHeap, 0, code); } virtual void allocMem( @@ -97,58 +91,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void ** coldCodeBlock, /* OUT */ void ** roDataBlock /* OUT */ ) { - //printf("allocMem\r\n"); - // TODO: Alignment? - //printf("Code size: %d\r\n", hotCodeSize); - auto code = HeapAlloc(m_executionEngine.m_codeHeap, 0, hotCodeSize); - *hotCodeBlock = m_codeAddr = code; - if (roDataSize != 0) { - // TODO: This mem needs to be freed... - *roDataBlock = m_dataAddr = GlobalAlloc(0, roDataSize); - } - } - - virtual void reserveUnwindInfo( - BOOL isFunclet, /* IN */ - BOOL isColdCode, /* IN */ - ULONG unwindSize /* IN */ - ) { - //printf("reserveUnwindInfo\r\n"); - } - - virtual void allocUnwindInfo( - BYTE * pHotCode, /* IN */ - BYTE * pColdCode, /* IN */ - ULONG startOffset, /* IN */ - ULONG endOffset, /* IN */ - ULONG unwindSize, /* IN */ - BYTE * pUnwindBlock, /* IN */ - CorJitFuncKind funcKind /* IN */ - ) { - //printf("allocUnwindInfo\r\n"); - } - - virtual void * allocGCInfo( - size_t size /* IN */ - ) { - //printf("allocGCInfo\r\n"); - return malloc(size); - } - - virtual void yieldExecution() { - } - - virtual void setEHcount( - unsigned cEH /* IN */ - ) { - printf("setEHcount\r\n"); - } - - virtual void setEHinfo( - unsigned EHnumber, /* IN */ - const CORINFO_EH_CLAUSE *clause /* IN */ - ) { - printf("setEHinfo\r\n"); } virtual BOOL logMsg(unsigned level, const char* fmt, va_list args) { @@ -169,24 +111,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { printf("Fatal error %X\r\n", result); } - virtual HRESULT allocBBProfileBuffer( - ULONG count, // The number of basic blocks that we have - ProfileBuffer ** profileBuffer - ) { - printf("Alloc bb profile buffer\r\n"); - return E_FAIL; - } - - virtual HRESULT getBBProfileData( - CORINFO_METHOD_HANDLE ftnHnd, - ULONG * count, // The number of basic blocks that we have - ProfileBuffer ** profileBuffer, - ULONG * numRuns - ) { - printf("getBBProfileData\r\n"); - return E_FAIL; - } - #if !defined(RYUJIT_CTPBUILD) // Associates a native call site, identified by its offset in the native code stream, with // the signature information and method handle the JIT used to lay out the call site. If @@ -626,66 +550,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return IAT_VALUE; } - // (static fields only) given that 'field' refers to thread local store, - // return the ID (TLS index), which is used to find the begining of the - // TLS data area for the particular DLL 'field' is associated with. - virtual DWORD getFieldThreadLocalStoreID( - CORINFO_FIELD_HANDLE field, - void **ppIndirection = NULL - ) { - printf("getFieldThreadLocalStoreID\r\n"); return 0; - } - - // Sets another object to intercept calls to "self" and current method being compiled - virtual void setOverride( - ICorDynamicInfo *pOverride, - CORINFO_METHOD_HANDLE currentMethod - ) { - printf("setOverride\r\n"); - } - - // Adds an active dependency from the context method's module to the given module - // This is internal callback for the EE. JIT should not call it directly. - virtual void addActiveDependency( - CORINFO_MODULE_HANDLE moduleFrom, - CORINFO_MODULE_HANDLE moduleTo - ) { - printf("addActiveDependency\r\n"); - } - - virtual CORINFO_METHOD_HANDLE GetDelegateCtor( - CORINFO_METHOD_HANDLE methHnd, - CORINFO_CLASS_HANDLE clsHnd, - CORINFO_METHOD_HANDLE targetMethodHnd, - DelegateCtorArgs * pCtorData - ) { - printf("GetDelegateCtor\r\n"); return nullptr; - } - - virtual void MethodCompileComplete( - CORINFO_METHOD_HANDLE methHnd - ) { - printf("MethodCompileComplete\r\n"); - } - - // return a thunk that will copy the arguments for the given signature. - virtual void* getTailCallCopyArgsThunk( - CORINFO_SIG_INFO *pSig, - CorInfoHelperTailCallSpecialHandling flags - ) { - printf("getTailCallCopyArgsThunk\r\n"); - return nullptr; - } - - /* ICorStaticInfo */ - virtual bool getSystemVAmd64PassStructInRegisterDescriptor( - /* IN */ CORINFO_CLASS_HANDLE structHnd, - /* OUT */ SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* structPassInRegDescPtr - ) { - assert(false); - return false; - } - // return flags (defined above, CORINFO_FLG_PUBLIC ...) virtual DWORD getMethodAttribs( CORINFO_METHOD_HANDLE ftn /* IN */ @@ -891,15 +755,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } - // Indicates if the method is an instance of the generic - // method that passes (or has passed) verification - virtual CorInfoInstantiationVerification isInstantiationOfVerifiedGeneric( - CORINFO_METHOD_HANDLE method /* IN */ - ) { - //printf("isInstantiationOfVerifiedGeneric\r\n"); - return INSTVER_NOT_INSTANTIATION; - } - // Loads the constraints on a typical method definition, detecting cycles; // for use in verification. virtual void initConstraintsForVerification( @@ -912,15 +767,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { //printf("initConstraintsForVerification\r\n"); } - // Returns enum whether the method does not require verification - // Also see ICorModuleInfo::canSkipVerification - virtual CorInfoCanSkipVerificationResult canSkipMethodVerification( - CORINFO_METHOD_HANDLE ftnHandle - ) { - //printf("canSkipMethodVerification\r\n"); - return CORINFO_VERIFICATION_CAN_SKIP; - } - // load and restore the method virtual void methodMustBeLoadedBeforeCodeIsRun( CORINFO_METHOD_HANDLE method @@ -1105,15 +951,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // This is only used by ngen for calculating certain hints. // - // Returns enum whether the module does not require verification - // Also see ICorMethodInfo::canSkipMethodVerification(); - virtual CorInfoCanSkipVerificationResult canSkipVerification( - CORINFO_MODULE_HANDLE module /* IN */ - ) { - printf("canSkipVerification\r\n"); - return CORINFO_VERIFICATION_CAN_SKIP; - } - // Checks if the given metadata token is valid virtual BOOL isValidToken( CORINFO_MODULE_HANDLE module, /* IN */ @@ -1768,14 +1605,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return NULL; } - // Returns type of HFA for valuetype - virtual CorInfoType getHFAType( - CORINFO_CLASS_HANDLE hClass - ) { - printf("getHFAType\r\n"); - return CORINFO_TYPE_UNDEF; - } - /***************************************************************************** * ICorErrorInfo contains methods to deal with SEH exceptions being thrown * from the corinfo interface. These methods may be called when an exception @@ -1950,11 +1779,11 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { #endif // RYUJIT_CTPBUILD - void CorJitInfo::getAddressOfPInvokeTarget(CORINFO_METHOD_HANDLE method, CORINFO_CONST_LOOKUP * pLookup) + void getAddressOfPInvokeTarget(CORINFO_METHOD_HANDLE method, CORINFO_CONST_LOOKUP * pLookup) { } - DWORD CorJitInfo::getJitFlags(CORJIT_FLAGS * flags, DWORD sizeInBytes) + DWORD getJitFlags(CORJIT_FLAGS * flags, DWORD sizeInBytes) { return 0; } From 6985362867799bd69ea9d1e4c67a5fba564fe821 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 07:10:04 +1000 Subject: [PATCH 016/273] More portability changes. Create hashable maps --- Pyjion/ilgen.h | 43 ++++++++++++++++++++++++------------------- Pyjion/jitinfo.h | 32 +------------------------------- Pyjion/pycomp.cpp | 4 +++- 3 files changed, 28 insertions(+), 51 deletions(-) diff --git a/Pyjion/ilgen.h b/Pyjion/ilgen.h index 2efa28214..e4e9b433c 100644 --- a/Pyjion/ilgen.h +++ b/Pyjion/ilgen.h @@ -27,7 +27,7 @@ #define ILGEN_H #define FEATURE_NO_HOST -#define USE_STL + #include #include #include @@ -62,15 +62,20 @@ class LabelInfo { } }; +struct CorInfoTypeHash { + std::size_t operator()(CorInfoType e) const { + return static_cast(e); + } +}; class ILGenerator { vector m_params, m_locals; CorInfoType m_retType; Module* m_module; - unordered_map> m_freedLocals; + unordered_map, CorInfoTypeHash> m_freedLocals; public: - vector m_il; + vector m_il; int m_localCount; vector m_labels; @@ -142,7 +147,7 @@ class ILGenerator { void localloc() { push_back(CEE_PREFIX1); - push_back((byte)CEE_LOCALLOC); + push_back((BYTE)CEE_LOCALLOC); } void ret() { @@ -176,7 +181,7 @@ class ILGenerator { } else { m_il.push_back(CEE_LDC_I4); - m_il.push_back((byte)CEE_STLOC); + m_il.push_back((BYTE)CEE_STLOC); emit_int(i); } } @@ -240,7 +245,7 @@ class ILGenerator { m_il.push_back(CEE_BNE_UN_S); break; } - m_il.push_back((byte)offset - 2); + m_il.push_back((BYTE)offset - 2); } else { switch (branchType) { @@ -285,7 +290,7 @@ class ILGenerator { void compare_eq() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CEQ); + m_il.push_back((BYTE)CEE_CEQ); } void compare_ne() { @@ -296,38 +301,38 @@ class ILGenerator { void compare_gt() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CGT); + m_il.push_back((BYTE)CEE_CGT); } void compare_lt() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CLT); + m_il.push_back((BYTE)CEE_CLT); } void compare_ge() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CLT); + m_il.push_back((BYTE)CEE_CLT); ld_i4(0); compare_eq(); } void compare_le() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CGT); + m_il.push_back((BYTE)CEE_CGT); ld_i4(0); compare_eq(); } void compare_ge_float() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CLT_UN); + m_il.push_back((BYTE)CEE_CLT_UN); ld_i4(0); compare_eq(); } void compare_le_float() { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_CGT_UN); + m_il.push_back((BYTE)CEE_CGT_UN); ld_i4(0); compare_eq(); } @@ -408,7 +413,7 @@ class ILGenerator { } else { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_STLOC); + m_il.push_back((BYTE)CEE_STLOC); m_il.push_back(index & 0xff); m_il.push_back((index >> 8) & 0xff); } @@ -429,7 +434,7 @@ class ILGenerator { } else { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_LDLOC); + m_il.push_back((BYTE)CEE_LDLOC); m_il.push_back(index & 0xff); m_il.push_back((index >> 8) & 0xff); } @@ -444,7 +449,7 @@ class ILGenerator { } else { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_LDLOCA); + m_il.push_back((BYTE)CEE_LDLOCA); m_il.push_back(index & 0xff); m_il.push_back((index >> 8) & 0xff); } @@ -479,7 +484,7 @@ class ILGenerator { CorJitResult result = jit->compileMethod( /*ICorJitInfo*/jitInfo, /*CORINFO_METHOD_INFO */&methodInfo, - /*flags*/CORJIT_FLG_SKIP_VERIFICATION, + /*flags*/ CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_SKIP_VERIFICATION , &nativeEntry, &nativeSizeOfCode ); @@ -523,7 +528,7 @@ class ILGenerator { } else { m_il.push_back(CEE_PREFIX1); - m_il.push_back((byte)CEE_LDARG); + m_il.push_back((BYTE)CEE_LDARG); m_il.push_back(index & 0xff); m_il.push_back((index >> 8) & 0xff); } @@ -539,7 +544,7 @@ class ILGenerator { m_il.push_back((value >> 24) & 0xff); } - void push_back(byte b) { + void push_back(BYTE b) { m_il.push_back(b); } diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index a0fd17846..fb7d6c9b6 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -68,7 +68,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { freeMem(m_codeAddr); } if (m_dataAddr != nullptr) { - ::GlobalFree(m_dataAddr); + free(m_dataAddr); } delete m_module; } @@ -190,13 +190,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return -1; } - virtual void getModuleNativeEntryPointRange( - void ** pStart, /* OUT */ - void ** pEnd /* OUT */ - ) { - printf("getModuleNativeEntryPointRange\r\n"); - } - // For what machine does the VM expect the JIT to generate code? The VM // returns one of the IMAGE_FILE_MACHINE_* values. Note that if the VM // is cross-compiling (such as the case for crossgen), it will return a @@ -263,29 +256,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return 0; } - static void ThrowFunc() { - } - - static void FailFast() { - ::ExitProcess(1); - } - - // return the native entry point to an EE helper (see CorInfoHelpFunc) - virtual void* getHelperFtn( - CorInfoHelpFunc ftnNum, - void **ppIndirection = NULL - ) { - switch (ftnNum) { - case CORINFO_HELP_THROW: return &ThrowFunc; - case CORINFO_HELP_FAIL_FAST: return &FailFast; - case CORINFO_HELP_DBLREM: - auto res = (double(*)(double, double))&fmod; - return res; - } - printf("unknown getHelperFtn\r\n"); - return NULL; - } - // return a callable address of the function (native code). This function // may return a different value (depending on whether the method has // been JITed or not. diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 27f133aa4..3fb1fb108 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -23,13 +23,15 @@ * */ -#include + #include #include #include #include "pycomp.h" +using namespace std; + HRESULT __stdcall GetCORSystemDirectoryInternal(SString& pbuffer) { printf("get cor system\n"); return S_OK; From 9c3eb6887667adf6449e53cb272c23ad9765f2a9 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 07:22:06 +1000 Subject: [PATCH 017/273] More updates to interfaces in underlying .NET APIs --- Pyjion/cee.h | 23 ++++++++++++++---- Pyjion/jitinfo.h | 2 +- Pyjion/pycomp.cpp | 62 ++++++----------------------------------------- 3 files changed, 27 insertions(+), 60 deletions(-) diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 58300519b..dadb4547f 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -51,28 +51,41 @@ using namespace std; class CCorJitHost : public ICorJitHost { - void * allocateMemory(size_t size, bool usePageAllocator = false) + void * allocateMemory(size_t size) override { + // TODO return nullptr; } - void freeMemory(void * block, bool usePageAllocator = false) + void freeMemory(void * block) override { + // TODO.. } - int getIntConfigValue(const wchar_t * name, int defaultValue) + int getIntConfigValue(const WCHAR* name, int defaultValue) override { return defaultValue; } - const wchar_t * getStringConfigValue(const wchar_t * name) + const WCHAR * getStringConfigValue(const WCHAR* name) override { return nullptr; } - void freeStringConfigValue(const wchar_t * value) + void freeStringConfigValue(const WCHAR* value) override { } + + void* allocateSlab(size_t size, size_t* pActualSize) override + { + *pActualSize = size; + return allocateMemory(size); + } + + void freeSlab(void* slab, size_t actualSize) override + { + freeMemory(slab); + } }; void CeeInit(); diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index fb7d6c9b6..fdb57883b 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -63,7 +63,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { m_module = module; } - ~CorJitInfo() { + ~CorJitInfo() override { if (m_codeAddr != nullptr) { freeMem(m_codeAddr); } diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 3fb1fb108..27b7d72e0 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -32,62 +32,16 @@ using namespace std; -HRESULT __stdcall GetCORSystemDirectoryInternal(SString& pbuffer) { - printf("get cor system\n"); - return S_OK; -} - -CExecutionEngine g_execEngine; - -extern "C" __declspec(dllexport) BOOL WINAPI DllMain( - _In_ HINSTANCE hinstDLL, - _In_ DWORD fdwReason, - _In_ LPVOID lpvReserved - ) { - switch (fdwReason) { - case DLL_PROCESS_ATTACH: break; - case DLL_PROCESS_DETACH: break; - case DLL_THREAD_ATTACH: break; - case DLL_THREAD_DETACH: - g_execEngine.TLS_ThreadDetaching(); - break; - } - return TRUE; -} - -BOOL EEHeapFreeInProcessHeap(DWORD dwFlags, LPVOID lpMem) { - return ::HeapFree(g_execEngine.ClrGetProcessHeap(), dwFlags, lpMem); -} - -LPVOID EEHeapAllocInProcessHeap(DWORD dwFlags, SIZE_T dwBytes) { - return ::HeapAlloc(g_execEngine.ClrGetProcessHeap(), dwFlags, dwBytes); -} - -void* __stdcall GetCLRFunction(LPCSTR functionName) { - if (strcmp(functionName, "EEHeapAllocInProcessHeap") == 0) { - return ::EEHeapAllocInProcessHeap; - } - else if (strcmp(functionName, "EEHeapFreeInProcessHeap") == 0) { - return ::EEHeapFreeInProcessHeap; - } - printf("get clr function %s\n", functionName); - return NULL; -} - -IExecutionEngine* __stdcall IEE() { - return &g_execEngine; -} - CCorJitHost g_jitHost; void CeeInit() { - CoreClrCallbacks cccallbacks; - cccallbacks.m_hmodCoreCLR = (HINSTANCE)GetModuleHandleW(NULL); - cccallbacks.m_pfnIEE = IEE; - cccallbacks.m_pfnGetCORSystemDirectory = GetCORSystemDirectoryInternal; - cccallbacks.m_pfnGetCLRFunction = GetCLRFunction; - - InitUtilcode(cccallbacks); +// CoreClrCallbacks cccallbacks; +// cccallbacks.m_hmodCoreCLR = (HINSTANCE)GetModuleHandleW(NULL); +// cccallbacks.m_pfnIEE = IEE; +// cccallbacks.m_pfnGetCORSystemDirectory = GetCORSystemDirectoryInternal; +// cccallbacks.m_pfnGetCLRFunction = GetCLRFunction; +// +// InitUtilcode(cccallbacks); // TODO: We should re-enable contracts and handle exceptions from OOM // and just fail the whole compilation if we hit that. Right now we // just leak an exception out across the JIT boundary. @@ -1189,7 +1143,7 @@ void PythonCompiler::emit_periodic_work() { } JittedCode* PythonCompiler::emit_compile() { - CorJitInfo* jitInfo = new CorJitInfo(g_execEngine, m_code, m_module); + CorJitInfo* jitInfo = new CorJitInfo(m_code, m_module); auto addr = m_il.compile(jitInfo, g_jit, m_code->co_stacksize + 100).m_addr; if (addr == nullptr) { printf("Compiling failed %s from %s line %d\r\n", From 37ad67012e8ae693c83406b6e8a68cba6ab4a797 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 07:25:58 +1000 Subject: [PATCH 018/273] Implement pure virtual methods of CEE host --- Pyjion/jitinfo.h | 195 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index fdb57883b..9a883e83e 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -1758,6 +1758,201 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return 0; } + void getMethodVTableOffset(CORINFO_METHOD_HANDLE method, unsigned int *offsetOfIndirection, + unsigned int *offsetAfterIndirection, bool *isRelative) override { + + } + + CORINFO_METHOD_HANDLE + resolveVirtualMethod(CORINFO_METHOD_HANDLE virtualMethod, CORINFO_CLASS_HANDLE implementingClass, + CORINFO_CONTEXT_HANDLE ownerType) override { + return nullptr; + } + + CORINFO_METHOD_HANDLE getUnboxedEntry(CORINFO_METHOD_HANDLE ftn, bool *requiresInstMethodTableArg) override { + return nullptr; + } + + CORINFO_CLASS_HANDLE getDefaultEqualityComparerClass(CORINFO_CLASS_HANDLE elemType) override { + return nullptr; + } + + void + expandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN *pResolvedToken, CORINFO_GENERICHANDLE_RESULT *pResult) override { + + } + + void setPatchpointInfo(PatchpointInfo *patchpointInfo) override { + + } + + PatchpointInfo *getOSRInfo(unsigned int *ilOffset) override { + return nullptr; + } + + bool tryResolveToken(CORINFO_RESOLVED_TOKEN *pResolvedToken) override { + return false; + } + + LPCWSTR getStringLiteral(CORINFO_MODULE_HANDLE module, unsigned int metaTOK, int *length) override { + return nullptr; + } + + const char *getClassNameFromMetadata(CORINFO_CLASS_HANDLE cls, const char **namespaceName) override { + return nullptr; + } + + CORINFO_CLASS_HANDLE getTypeInstantiationArgument(CORINFO_CLASS_HANDLE cls, unsigned int index) override { + return nullptr; + } + + CorInfoInlineTypeCheck canInlineTypeCheck(CORINFO_CLASS_HANDLE cls, CorInfoInlineTypeCheckSource source) override { + return CORINFO_INLINE_TYPECHECK_USE_HELPER; + } + + unsigned int getHeapClassSize(CORINFO_CLASS_HANDLE cls) override { + return 0; + } + + BOOL canAllocateOnStack(CORINFO_CLASS_HANDLE cls) override { + return false; + } + + CorInfoHelpFunc getNewHelper(CORINFO_RESOLVED_TOKEN *pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, + bool *pHasSideEffects) override { + return CORINFO_HELP_GETREFANY; + } + + bool getReadyToRunHelper(CORINFO_RESOLVED_TOKEN *pResolvedToken, CORINFO_LOOKUP_KIND *pGenericLookupKind, + CorInfoHelpFunc id, CORINFO_CONST_LOOKUP *pLookup) override { + return false; + } + + void getReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN *pTargetMethod, CORINFO_CLASS_HANDLE delegateType, + CORINFO_LOOKUP *pLookup) override { + + } + + CorInfoInitClassResult + initClass(CORINFO_FIELD_HANDLE field, CORINFO_METHOD_HANDLE method, CORINFO_CONTEXT_HANDLE context) override { + return CORINFO_INITCLASS_INITIALIZED; + } + + CorInfoType getTypeForPrimitiveNumericClass(CORINFO_CLASS_HANDLE cls) override { + return CORINFO_TYPE_BYREF; + } + + TypeCompareState compareTypesForCast(CORINFO_CLASS_HANDLE fromClass, CORINFO_CLASS_HANDLE toClass) override { + return TypeCompareState::May; + } + + TypeCompareState compareTypesForEquality(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2) override { + return TypeCompareState::May; + } + + BOOL isMoreSpecificType(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2) override { + return false; + } + + void *allocateArray(size_t cBytes) override { + return nullptr; + } + + CorInfoHFAElemType getHFAType(CORINFO_CLASS_HANDLE hClass) override { + return CORINFO_HFA_ELEM_DOUBLE; + } + + bool runWithErrorTrap(void (*function)(void *), void *parameter) override { + return false; + } + + const char *getMethodNameFromMetadata(CORINFO_METHOD_HANDLE ftn, const char **className, const char **namespaceName, + const char **enclosingClassName) override { + return nullptr; + } + + bool getSystemVAmd64PassStructInRegisterDescriptor(CORINFO_CLASS_HANDLE structHnd, + SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR *structPassInRegDescPtr) override { + return false; + } + + void *getHelperFtn(CorInfoHelpFunc ftnNum, void **ppIndirection) override { + return nullptr; + } + + void getLocationOfThisType(CORINFO_METHOD_HANDLE context, CORINFO_LOOKUP_KIND *pLookupKind) override { + + } + + CORINFO_CLASS_HANDLE getStaticFieldCurrentClass(CORINFO_FIELD_HANDLE field, bool *pIsSpeculative) override { + return nullptr; + } + + DWORD getFieldThreadLocalStoreID(CORINFO_FIELD_HANDLE field, void **ppIndirection) override { + return 0; + } + + void setOverride(ICorDynamicInfo *pOverride, CORINFO_METHOD_HANDLE currentMethod) override { + + } + + void addActiveDependency(CORINFO_MODULE_HANDLE moduleFrom, CORINFO_MODULE_HANDLE moduleTo) override { + + } + + CORINFO_METHOD_HANDLE + GetDelegateCtor(CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, + DelegateCtorArgs *pCtorData) override { + return nullptr; + } + + void MethodCompileComplete(CORINFO_METHOD_HANDLE methHnd) override { + + } + + bool getTailCallHelpers(CORINFO_RESOLVED_TOKEN *callToken, CORINFO_SIG_INFO *sig, + CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS *pResult) override { + return false; + } + + bool convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN *pResolvedToken, bool fMustConvert) override { + return false; + } + + void notifyInstructionSetUsage(CORINFO_InstructionSet instructionSet, bool supportEnabled) override { + + } + + void reserveUnwindInfo(BOOL isFunclet, BOOL isColdCode, ULONG unwindSize) override { + + } + + void allocUnwindInfo(BYTE *pHotCode, BYTE *pColdCode, ULONG startOffset, ULONG endOffset, ULONG unwindSize, + BYTE *pUnwindBlock, CorJitFuncKind funcKind) override { + + } + + void *allocGCInfo(size_t size) override { + return nullptr; + } + + void setEHcount(unsigned int cEH) override { + + } + + void setEHinfo(unsigned int EHnumber, const CORINFO_EH_CLAUSE *clause) override { + + } + + HRESULT allocMethodBlockCounts(UINT32 count, BlockCounts **pBlockCounts) override { + return 0; + } + + HRESULT getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd, UINT32 *pCount, BlockCounts **pBlockCounts, + UINT32 *pNumRuns) override { + return 0; + } + }; #endif From 2312f937b9ecaf83d6b967f66f89708ce20b698f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 07:30:23 +1000 Subject: [PATCH 019/273] Hash method updates --- Pyjion/absint.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 01e403a01..65ab57e9c 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -38,6 +38,12 @@ #define GET_OPARG(index) _Py_OPARG(m_byteCode[index/sizeof(_Py_CODEUNIT)]) #define GET_OPCODE(index) _Py_OPCODE(m_byteCode[index/sizeof(_Py_CODEUNIT)]) +struct AbstractValueKindHash { + std::size_t operator()(AbstractValueKind e) const { + return static_cast(e); + } +}; + AbstractInterpreter::AbstractInterpreter(PyCodeObject *code, IPythonCompiler* comp) : m_code(code), m_comp(comp) { m_byteCode = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code); m_size = PyBytes_Size(code->co_code); @@ -3591,7 +3597,7 @@ LocalKind get_optimized_local_kind(AbstractValueKind kind) { Local AbstractInterpreter::get_optimized_local(int index, AbstractValueKind kind) { if (m_optLocals.find(index) == m_optLocals.end()) { - m_optLocals[index] = unordered_map(); + m_optLocals[index] = unordered_map(); } auto& map = m_optLocals.find(index)->second; if (map.find(kind) == map.end()) { From 0368ab4b3d8e80ddbe3692f241df19125fe97262 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:08:26 +1000 Subject: [PATCH 020/273] Remove more dead code and add explicit cast to void* in GlobalMethod macros --- Pyjion/codemodel.h | 3 --- Pyjion/pycomp.cpp | 27 ++------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/Pyjion/codemodel.h b/Pyjion/codemodel.h index b5271a7ba..791a0a00d 100644 --- a/Pyjion/codemodel.h +++ b/Pyjion/codemodel.h @@ -112,9 +112,6 @@ class Method : public BaseMethod { CorInfoType m_retType; void* m_addr; - Method() { - } - Method(Module* module, CorInfoType returnType, std::vector params, void* addr) { m_retType = returnType; m_params = params; diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 27b7d72e0..153c99607 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -35,17 +35,6 @@ using namespace std; CCorJitHost g_jitHost; void CeeInit() { -// CoreClrCallbacks cccallbacks; -// cccallbacks.m_hmodCoreCLR = (HINSTANCE)GetModuleHandleW(NULL); -// cccallbacks.m_pfnIEE = IEE; -// cccallbacks.m_pfnGetCORSystemDirectory = GetCORSystemDirectoryInternal; -// cccallbacks.m_pfnGetCLRFunction = GetCLRFunction; -// -// InitUtilcode(cccallbacks); - // TODO: We should re-enable contracts and handle exceptions from OOM - // and just fail the whole compilation if we hit that. Right now we - // just leak an exception out across the JIT boundary. - jitStartup(&g_jitHost); #if _DEBUG DisableThrowCheck(); @@ -61,8 +50,6 @@ class InitHolder { InitHolder g_initHolder; -//#define DEBUG_TRACE - Module g_module; ICorJitCompiler* g_jit; @@ -143,16 +130,6 @@ void PythonCompiler::emit_incref(bool maybeTagged) { void PythonCompiler::decref() { m_il.emit_call(METHOD_DECREF_TOKEN); - // It might be nice to inline this at some point: - //LD_FIELDA(PyObject, ob_refcnt); - //m_il.push_back(CEE_DUP); - //m_il.push_back(CEE_LDIND_I4); - //m_il.push_back(CEE_LDC_I4_1); - //m_il.push_back(CEE_SUB); - ////m_il.push_back(CEE_DUP); - //// _Py_Dealloc(_py_decref_tmp) - - //m_il.push_back(CEE_STIND_I4); } @@ -1169,14 +1146,14 @@ void PythonCompiler::emit_tagged_int_to_float() { class GlobalMethod { Method m_method; public: - GlobalMethod(int token, Method method) { + GlobalMethod(int token, Method method) : m_method(method) { m_method = method; g_module.m_methods[token] = &m_method; } }; #define GLOBAL_METHOD(token, addr, returnType, ...) \ - GlobalMethod g ## token(token, Method(&g_module, returnType, std::vector{__VA_ARGS__}, addr)); + GlobalMethod g ## token(token, Method(&g_module, returnType, std::vector{__VA_ARGS__}, (void*)addr)); GLOBAL_METHOD(METHOD_ADD_TOKEN, &PyJit_Add, CORINFO_TYPE_NATIVEINT, Parameter(CORINFO_TYPE_NATIVEINT), Parameter(CORINFO_TYPE_NATIVEINT)); GLOBAL_METHOD(METHOD_SUBSCR_TOKEN, &PyJit_Subscr, CORINFO_TYPE_NATIVEINT, Parameter(CORINFO_TYPE_NATIVEINT), Parameter(CORINFO_TYPE_NATIVEINT)); From a64e3daac8f035eb19219a31d7e66ddb8a29b060 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:10:34 +1000 Subject: [PATCH 021/273] Fixup hashable unordered map types in interpreter --- Pyjion/absint.cpp | 5 ----- Pyjion/absint.h | 8 +++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 65ab57e9c..b6e1c6a8c 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -38,11 +38,6 @@ #define GET_OPARG(index) _Py_OPARG(m_byteCode[index/sizeof(_Py_CODEUNIT)]) #define GET_OPCODE(index) _Py_OPCODE(m_byteCode[index/sizeof(_Py_CODEUNIT)]) -struct AbstractValueKindHash { - std::size_t operator()(AbstractValueKind e) const { - return static_cast(e); - } -}; AbstractInterpreter::AbstractInterpreter(PyCodeObject *code, IPythonCompiler* comp) : m_code(code), m_comp(comp) { m_byteCode = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code); diff --git a/Pyjion/absint.h b/Pyjion/absint.h index 5a2049bac..beb774332 100644 --- a/Pyjion/absint.h +++ b/Pyjion/absint.h @@ -84,6 +84,12 @@ enum EhFlags { EhFlags operator | (EhFlags lhs, EhFlags rhs); EhFlags operator |= (EhFlags& lhs, EhFlags rhs); +struct AbstractValueKindHash { + std::size_t operator()(AbstractValueKind e) const { + return static_cast(e); + } +}; + struct ExceptionVars { // The previous exception value before we took the exception we're currently // handling. These correspond with the values in tstate->exc_* and will be @@ -221,7 +227,7 @@ class __declspec(dllexport) AbstractInterpreter { // unpack. unordered_map m_sequenceLocals; unordered_map m_assignmentState; - unordered_map> m_optLocals; + unordered_map> m_optLocals; #pragma warning (default:4251) From 1b897dd2cce57c928496b9be30b88d65a9728fb7 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:22:44 +1000 Subject: [PATCH 022/273] Link JIT libraries in directly --- CMakeLists.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2591bea7f..41d9cc0db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,18 +28,17 @@ if(NOT WIN32) endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif(NOT WIN32) +include_directories(CoreCLR/src/coreclr/src/inc) + if (APPLE) include_directories(Pyjion/compat) - add_definitions(-DHOST_UNIX) - add_definitions(-DHOST_AMD64) - add_definitions(-DHOST_64BIT) + include_directories(CoreCLR/artifacts/bin/coreclr/OSX.x64.Debug/inc) set(CLR_CMAKE_HOST_UNIX 1) endif() -include_directories(CoreCLR/src/coreclr/src/inc) -include_directories(CoreCLR/artifacts/bin/coreclr/OSX.x64.Debug/inc) - - set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) + add_library(pyjion SHARED ${SOURCES}) +target_link_libraries(pyjion ${Python3_LIBRARIES}) +target_link_libraries(pyjion ${CMAKE_SOURCE_DIR}/CoreCLR/artifacts/bin/coreclr/OSX.x64.Debug/libclrjit.dylib) add_compile_options(-Wno-sign-compare) From 34011adbec39fb582ee6088a51eb8fb1d4f59a2a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:24:11 +1000 Subject: [PATCH 023/273] Remove all the patches that are no longer needed --- PatchDeps.bat | 17 --------- PatchDeps.sh | 13 ------- Patches/CoreCLR/CMakeLists.txt | 20 ---------- Patches/CoreCLR/DiffCoreCLR.bat | 20 ---------- Patches/CoreCLR/build.cmd | 13 ------- Patches/CoreCLR/clr.coreclr.props | 38 ------------------- Patches/CoreCLR/clr.defines.targets | 13 ------- Patches/CoreCLR/clr.desktop.props | 13 ------- Patches/CoreCLR/src/CMakeLists.txt | 32 ---------------- Patches/CoreCLR/src/inc/utilcode.h | 13 ------- Patches/CoreCLR/src/utilcode/CMakeLists.txt | 10 ----- .../src/utilcode/longfilepathwrappers.cpp | 26 ------------- Patches/CoreCLR/src/utilcode/util.cpp | 20 ---------- 13 files changed, 248 deletions(-) delete mode 100644 PatchDeps.bat delete mode 100644 PatchDeps.sh delete mode 100644 Patches/CoreCLR/CMakeLists.txt delete mode 100644 Patches/CoreCLR/DiffCoreCLR.bat delete mode 100644 Patches/CoreCLR/build.cmd delete mode 100644 Patches/CoreCLR/clr.coreclr.props delete mode 100644 Patches/CoreCLR/clr.defines.targets delete mode 100644 Patches/CoreCLR/clr.desktop.props delete mode 100644 Patches/CoreCLR/src/CMakeLists.txt delete mode 100644 Patches/CoreCLR/src/inc/utilcode.h delete mode 100644 Patches/CoreCLR/src/utilcode/CMakeLists.txt delete mode 100644 Patches/CoreCLR/src/utilcode/longfilepathwrappers.cpp delete mode 100644 Patches/CoreCLR/src/utilcode/util.cpp diff --git a/PatchDeps.bat b/PatchDeps.bat deleted file mode 100644 index 18f9161c5..000000000 --- a/PatchDeps.bat +++ /dev/null @@ -1,17 +0,0 @@ -@echo off -REM ######################################################## -REM # Apply changes to disable COM interop support - -echo Disabling COM interop support in CoreCLR... -pushd CoreCLR -git apply ..\Patches\CoreCLR\src\inc\utilcode.h -git apply ..\Patches\CoreCLR\src\utilcode\CMakeLists.txt -git apply ..\Patches\CoreCLR\src\utilcode\longfilepathwrappers.cpp -git apply ..\Patches\CoreCLR\src\utilcode\util.cpp -git apply ..\Patches\CoreCLR\src\CMakeLists.txt -git apply ..\Patches\CoreCLR\build.cmd -git apply ..\Patches\CoreCLR\clr.coreclr.props -git apply ..\Patches\CoreCLR\clr.defines.targets -git apply ..\Patches\CoreCLR\clr.desktop.props -git apply ..\Patches\CoreCLR\CMakeLists.txt -popd diff --git a/PatchDeps.sh b/PatchDeps.sh deleted file mode 100644 index 871bbc44e..000000000 --- a/PatchDeps.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -echo Patching CoreCLR.... -pushd CoreCLR -git apply ../Patches/CoreCLR/src/inc/utilcode.h -git apply ../Patches/CoreCLR/src/utilcode/CMakeLists.txt -git apply ../Patches/CoreCLR/src/utilcode/util.cpp -git apply ../Patches/CoreCLR/src/CMakeLists.txt -git apply ../Patches/CoreCLR/build.cmd -git apply ../Patches/CoreCLR/clr.coreclr.props -git apply ../Patches/CoreCLR/clr.defines.targets -git apply ../Patches/CoreCLR/clr.desktop.props -git apply ../Patches/CoreCLR/CMakeLists.txt -popd diff --git a/Patches/CoreCLR/CMakeLists.txt b/Patches/CoreCLR/CMakeLists.txt deleted file mode 100644 index 2b819029c..000000000 --- a/Patches/CoreCLR/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index ba328f4..2837e0a 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -762,13 +762,13 @@ add_definitions(-DFEATURE_ASYNC_IO) - add_definitions(-DFEATURE_BCL_FORMATTING) - add_definitions(-DFEATURE_COLLECTIBLE_TYPES) - --if(WIN32) -+if(FALSE) - add_definitions(-DFEATURE_CLASSIC_COMINTEROP) - add_definitions(-DFEATURE_COMINTEROP) - add_definitions(-DFEATURE_COMINTEROP_APARTMENT_SUPPORT) - add_definitions(-DFEATURE_COMINTEROP_UNMANAGED_ACTIVATION) - add_definitions(-DFEATURE_COMINTEROP_WINRT_MANAGED_ACTIVATION) --endif(WIN32) -+endif(FALSE) - - add_definitions(-DFEATURE_CORECLR) - if (CLR_CMAKE_PLATFORM_UNIX) diff --git a/Patches/CoreCLR/DiffCoreCLR.bat b/Patches/CoreCLR/DiffCoreCLR.bat deleted file mode 100644 index 596a24e1f..000000000 --- a/Patches/CoreCLR/DiffCoreCLR.bat +++ /dev/null @@ -1,20 +0,0 @@ -@echo off -REM ######################################################## -echo Gathering changes to disable COM interop support in CoreCLR... -pushd ..\..\CoreCLR -git diff v1.0.0-rc2 src\inc\ex.h > ..\Patches\CoreCLR\src\inc\ex.h -git diff v1.0.0-rc2 src\inc\utilcode.h > ..\Patches\CoreCLR\src\inc\utilcode.h -git diff v1.0.0-rc2 src\utilcode\posterror.cpp > ..\Patches\CoreCLR\src\utilcode\posterror.cpp -git diff v1.0.0-rc2 src\utilcode\util.cpp > ..\Patches\CoreCLR\src\utilcode\util.cpp -git diff v1.0.0-rc2 src\utilcode\CMakeLists.txt > ..\Patches\CoreCLR\src\utilcode\CMakeLists.txt -git diff v1.0.0-rc2 src\vm\clrex.cpp > ..\Patches\CoreCLR\src\vm\clrex.cpp -git diff v1.0.0-rc2 src\vm\comcache.cpp > ..\Patches\CoreCLR\src\vm\comcache.cpp -git diff v1.0.0-rc2 src\vm\interoputil.cpp > ..\Patches\CoreCLR\src\vm\interoputil.cpp -git diff v1.0.0-rc2 src\vm\threads.cpp > ..\Patches\CoreCLR\src\vm\threads.cpp -git diff v1.0.0-rc2 src\CMakeLists.txt > ..\Patches\CoreCLR\src\CMakeLists.txt -git diff v1.0.0-rc2 build.cmd > ..\Patches\CoreCLR\build.cmd -git diff v1.0.0-rc2 clr.coreclr.props > ..\Patches\CoreCLR\clr.coreclr.props -git diff v1.0.0-rc2 clr.defines.targets > ..\Patches\CoreCLR\clr.defines.targets -git diff v1.0.0-rc2 clr.desktop.props > ..\Patches\CoreCLR\clr.desktop.props -git diff v1.0.0-rc2 CMakeLists.txt > ..\Patches\CoreCLR\CMakeLists.txt -popd diff --git a/Patches/CoreCLR/build.cmd b/Patches/CoreCLR/build.cmd deleted file mode 100644 index dba5f798f..000000000 --- a/Patches/CoreCLR/build.cmd +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/build.cmd b/build.cmd -index 82fa0a5..2f4f0d8 100644 ---- a/build.cmd -+++ b/build.cmd -@@ -338,7 +338,7 @@ set __msbuildLogArgs=^ - /consoleloggerparameters:Summary ^ - /verbosity:minimal - --set __msbuildArgs="%__IntermediatesDir%\install.vcxproj" %__msbuildCommonArgs% %__msbuildLogArgs% /p:Configuration=%__BuildType% -+set __msbuildArgs="%__IntermediatesDir%\install.vcxproj" %__msbuildCommonArgs% %__msbuildLogArgs% /p:Configuration=%__BuildType% /p:FeatureCominterop=false - - if /i "%__BuildArch%" == "arm64" ( - REM TODO, remove once we have msbuild support for this platform. diff --git a/Patches/CoreCLR/clr.coreclr.props b/Patches/CoreCLR/clr.coreclr.props deleted file mode 100644 index a9aa999f4..000000000 --- a/Patches/CoreCLR/clr.coreclr.props +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/clr.coreclr.props b/clr.coreclr.props -index a335486..510a296 100644 ---- a/clr.coreclr.props -+++ b/clr.coreclr.props -@@ -10,7 +10,7 @@ - true - true - true -- true -+ false - true - true - true -@@ -53,11 +53,11 @@ - true - - true -- true -- true -+ false -+ false - true -- true -- true -+ false -+ false - true - true - true -@@ -76,7 +76,7 @@ - true - true - true -- true -+ false - true - - $(CDefines);FEATURE_ICASTABLE - $(CDefines);FEATURE_COMINTEROP_APARTMENT_SUPPORT - $(CDefines);FEATURE_COMINTEROP_MANAGED_ACTIVATION diff --git a/Patches/CoreCLR/clr.desktop.props b/Patches/CoreCLR/clr.desktop.props deleted file mode 100644 index 297e0c078..000000000 --- a/Patches/CoreCLR/clr.desktop.props +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/clr.desktop.props b/clr.desktop.props -index 88a4d9f..8c23f32 100644 ---- a/clr.desktop.props -+++ b/clr.desktop.props -@@ -19,7 +19,7 @@ - true - true - true -- true -+ false - true - true - true diff --git a/Patches/CoreCLR/src/CMakeLists.txt b/Patches/CoreCLR/src/CMakeLists.txt deleted file mode 100644 index 960362cb2..000000000 --- a/Patches/CoreCLR/src/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index 4ae4a7c..b76de27 100644 ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -125,25 +125,12 @@ endif(CLR_CMAKE_PLATFORM_UNIX) - - add_subdirectory(utilcode) - add_subdirectory(gcinfo) --add_subdirectory(coreclr) - add_subdirectory(jit) --add_subdirectory(vm) --add_subdirectory(md) --add_subdirectory(debug) - add_subdirectory(inc) --add_subdirectory(strongname) --add_subdirectory(binder) --add_subdirectory(classlibnative) --add_subdirectory(dlls) --add_subdirectory(ToolBox) --add_subdirectory(tools) --add_subdirectory(unwinder) --add_subdirectory(ildasm) --add_subdirectory(ilasm) - --if(WIN32) -+if(FALSE) - add_subdirectory(ipcman) --endif(WIN32) -+endif(FALSE) - - if(CLR_CMAKE_PLATFORM_UNIX) - add_subdirectory(palrt) diff --git a/Patches/CoreCLR/src/inc/utilcode.h b/Patches/CoreCLR/src/inc/utilcode.h deleted file mode 100644 index 47d13d7a6..000000000 --- a/Patches/CoreCLR/src/inc/utilcode.h +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/inc/utilcode.h b/src/inc/utilcode.h -index b9985a8..cade9e5 100644 ---- a/src/inc/utilcode.h -+++ b/src/inc/utilcode.h -@@ -8,6 +8,8 @@ - // - //***************************************************************************** - -+#define SetErrorInfo __DoNotUseSetErrorInfo -+ - #ifndef __UtilCode_h__ - #define __UtilCode_h__ - diff --git a/Patches/CoreCLR/src/utilcode/CMakeLists.txt b/Patches/CoreCLR/src/utilcode/CMakeLists.txt deleted file mode 100644 index a5005b47f..000000000 --- a/Patches/CoreCLR/src/utilcode/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -diff --git a/src/utilcode/CMakeLists.txt b/src/utilcode/CMakeLists.txt -index 9d35a7e..02d878a 100644 ---- a/src/utilcode/CMakeLists.txt -+++ b/src/utilcode/CMakeLists.txt -@@ -134,4 +134,4 @@ endif(CLR_CMAKE_PLATFORM_UNIX) - add_subdirectory(dac) - add_subdirectory(dyncrt) - add_subdirectory(staticnohost) --add_subdirectory(crossgen) -+ diff --git a/Patches/CoreCLR/src/utilcode/longfilepathwrappers.cpp b/Patches/CoreCLR/src/utilcode/longfilepathwrappers.cpp deleted file mode 100644 index bada55e1e..000000000 --- a/Patches/CoreCLR/src/utilcode/longfilepathwrappers.cpp +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/src/utilcode/longfilepathwrappers.cpp b/src/utilcode/longfilepathwrappers.cpp -index 2b5a8b4..54c8e09 100644 ---- a/src/utilcode/longfilepathwrappers.cpp -+++ b/src/utilcode/longfilepathwrappers.cpp -@@ -1190,10 +1190,6 @@ FindFirstFileExWrapper( - - #ifndef FEATURE_PAL - --#if ! defined(DACCESS_COMPILE) && !defined(SELF_NO_HOST) --extern HINSTANCE g_pMSCorEE; --#endif// ! defined(DACCESS_COMPILE) && !defined(SELF_NO_HOST) -- - BOOL PAL_GetPALDirectoryWrapper(SString& pbuffer) - { - -@@ -1203,10 +1199,6 @@ BOOL PAL_GetPALDirectoryWrapper(SString& pbuffer) - DWORD dwPath; - HINSTANCE hinst = NULL; - --#if ! defined(DACCESS_COMPILE) && !defined(SELF_NO_HOST) -- hinst = g_pMSCorEE; --#endif// ! defined(DACCESS_COMPILE) && !defined(SELF_NO_HOST) -- - #ifndef CROSSGEN_COMPILE - _ASSERTE(hinst != NULL); - #endif diff --git a/Patches/CoreCLR/src/utilcode/util.cpp b/Patches/CoreCLR/src/utilcode/util.cpp deleted file mode 100644 index 58aa8571c..000000000 --- a/Patches/CoreCLR/src/utilcode/util.cpp +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/utilcode/util.cpp b/src/utilcode/util.cpp -index f801f94..98e5e5d 100644 ---- a/src/utilcode/util.cpp -+++ b/src/utilcode/util.cpp -@@ -212,6 +212,7 @@ typedef HRESULT __stdcall DLLGETCLASSOBJECT(REFCLSID rclsid, - EXTERN_C const IID _IID_IClassFactory = - {0x00000001, 0x0000, 0x0000, {0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}}; - -+#if FEATURE_COMINTEROP - // ---------------------------------------------------------------------------- - // FakeCoCreateInstanceEx - // -@@ -374,6 +375,7 @@ HRESULT FakeCoCallDllGetClassObject(REFCLSID rclsid, - - return hr; - } -+#endif - - #if USE_UPPER_ADDRESS - static BYTE * s_CodeMinAddr; // Preferred region to allocate the code in. From b670bb24df2dc574f8ed1e8ff35e357cd982af00 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:28:51 +1000 Subject: [PATCH 024/273] removed Python submodule --- .gitmodules | 4 ---- Python | 1 - 2 files changed, 5 deletions(-) delete mode 160000 Python diff --git a/.gitmodules b/.gitmodules index 83c03de48..e232265c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "Python"] - path = Python - url = https://github.com/python/cpython.git - branch = 3.9 [submodule "dotnet"] path = CoreCLR url = https://github.com/dotnet/runtime diff --git a/Python b/Python deleted file mode 160000 index bdf46bc7e..000000000 --- a/Python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bdf46bc7e12c08aaab676a2c947f03daffe035a7 From 9b968733eafc90492f760253b0776fd3c77b818b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:30:48 +1000 Subject: [PATCH 025/273] Make the linking OS-agnostic --- CMakeLists.txt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41d9cc0db..9d140aacc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,15 +30,27 @@ endif(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/inc) +if (WIN32) + set(CLR_OS_BUILD WIN.x64.Debug) + set(CLR_JIT_LIB "clrjit.dll") +endif() + +if (LINUX) + set(CLR_OS_BUILD OSX.x64.Debug) + set(CLR_JIT_LIB "libclrjit.so") +endif() + if (APPLE) + set(CLR_OS_BUILD OSX.x64.Debug) include_directories(Pyjion/compat) - include_directories(CoreCLR/artifacts/bin/coreclr/OSX.x64.Debug/inc) set(CLR_CMAKE_HOST_UNIX 1) + set(CLR_JIT_LIB "libclrjit.dylib") endif() +include_directories(CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/inc) set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) add_library(pyjion SHARED ${SOURCES}) target_link_libraries(pyjion ${Python3_LIBRARIES}) -target_link_libraries(pyjion ${CMAKE_SOURCE_DIR}/CoreCLR/artifacts/bin/coreclr/OSX.x64.Debug/libclrjit.dylib) +target_link_libraries(pyjion ${CMAKE_SOURCE_DIR}/CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/${CLR_JIT_LIB}) add_compile_options(-Wno-sign-compare) From ad785803f8b5b65b2274fa2f2d8ad1559efcf286 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 08:31:49 +1000 Subject: [PATCH 026/273] Remove legacy build scripts --- BuildDebugPython.bat | 13 ---- BuildDebugPython.sh | 5 -- BuildDeps.cmd | 61 --------------- DebugBuild.bat | 6 -- Pyjion.sln | 68 ---------------- Pyjion/Pyjion.vcxproj | 176 ------------------------------------------ 6 files changed, 329 deletions(-) delete mode 100644 BuildDebugPython.bat delete mode 100644 BuildDebugPython.sh delete mode 100644 BuildDeps.cmd delete mode 100644 DebugBuild.bat delete mode 100644 Pyjion.sln delete mode 100644 Pyjion/Pyjion.vcxproj diff --git a/BuildDebugPython.bat b/BuildDebugPython.bat deleted file mode 100644 index 843a03757..000000000 --- a/BuildDebugPython.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -echo Building CPython in debug mode for x64 ... -pushd Python\PCBuild - -call get_externals.bat -call .\build.bat -c Debug -p x64 - -echo Copying .lib and .dll files to appropriate places ... -copy amd64\python35_d.lib ..\..\Libs\Debug\x64\ -copy amd64\python35_d.dll ..\..\x64\Debug - -popd \ No newline at end of file diff --git a/BuildDebugPython.sh b/BuildDebugPython.sh deleted file mode 100644 index df3d089bc..000000000 --- a/BuildDebugPython.sh +++ /dev/null @@ -1,5 +0,0 @@ -echo Building CPython in debug mode for x64 ... -pushd Python/ -./configure --with-pydebug -make -s -j2 -popd \ No newline at end of file diff --git a/BuildDeps.cmd b/BuildDeps.cmd deleted file mode 100644 index e657a275d..000000000 --- a/BuildDeps.cmd +++ /dev/null @@ -1,61 +0,0 @@ -@echo off - -:: 32-bit builds not supported as they have not been tested. -set __BuildArch=x64 -set __BuildType=Debug - - -:Arg_Loop -if "%1" == "" goto ArgsDone -if /i "%1" == "/?" goto Usage - -if /i "%1" == "Debug" (set __BuildType=Debug&shift&goto Arg_Loop) -if /i "%1" == "Release" (set __BuildType=Release&shift&goto Arg_Loop) - -echo Invalid commandline argument: %1 -goto Usage - -:ArgsDone - -:BuildCoreCLR -pushd coreclr -call build.cmd %__BuildArch% %__BuildType% -IF ERRORLEVEL 1 goto Error - -mkdir ..\Libs\%__BuildType%\%__BuildArch%\ -copy bin\obj\Windows_NT.%__BuildArch%.%__BuildType%\src\jit\dll\%__BuildType%\clrjit.lib ..\Libs\%__BuildType%\%__BuildArch%\ -copy bin\obj\Windows_NT.%__BuildArch%.%__BuildType%\src\utilcode\dyncrt\%__BuildType%\utilcode.lib ..\Libs\%__BuildType%\%__BuildArch%\ -copy bin\obj\Windows_NT.%__BuildArch%.%__BuildType%\src\gcinfo\lib\%__BuildType%\gcinfo.lib ..\Libs\%__BuildType%\%__BuildArch%\ -popd - -:BuildPython -:: Keep BuildDebugPython.bat in sync w/ any relevant changes made here. -pushd Python\PCBuild - -call get_externals.bat -IF ERRORLEVEL 1 goto Error -call .\build.bat -c %__BuildType% -p %__BuildArch% -IF ERRORLEVEL 1 goto Error - -if /i "%__BuildArch%" == "x64" set arch=amd64 -set SUFFIX= -if /i "%__BuildType%" == "Debug" set SUFFIX=_d - -copy %arch%\python39%SUFFIX%.lib ..\..\Libs\%__BuildType%\%__BuildArch%\ -popd - -:Done -echo All built... -exit /b 0 - -:Error -echo something failed to build... -exit /b 1 - -:Usage -echo. -echo Usage: -echo %0 [BuildType] where: -echo. -echo BuildType can be: Debug, Release -exit /b 1 diff --git a/DebugBuild.bat b/DebugBuild.bat deleted file mode 100644 index 730891563..000000000 --- a/DebugBuild.bat +++ /dev/null @@ -1,6 +0,0 @@ -@echo off -echo Building Pyjion ... -MSBuild.exe .\Pyjion.sln /p:Configuration=Debug;Platform=x64 /m - -echo Copying x64\Debug\pyjit.dll to add the JIT to Python ... -copy x64\Debug\pyjit.dll Python\PCbuild\amd64 \ No newline at end of file diff --git a/Pyjion.sln b/Pyjion.sln deleted file mode 100644 index 9f9fda5f4..000000000 --- a/Pyjion.sln +++ /dev/null @@ -1,68 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.24704.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Pyjion", "Pyjion\Pyjion.vcxproj", "{0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Test", "Test\Test.vcxproj", "{721AF7E3-9330-4598-AB73-5AB993FE3F9C}" - ProjectSection(ProjectDependencies) = postProject - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62} = {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Tests", "Tests\Tests.vcxproj", "{ADFE2428-2595-4535-822D-4E70405F7D58}" - ProjectSection(ProjectDependencies) = postProject - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62} = {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62} - EndProjectSection -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Mixed Platforms = Debug|Mixed Platforms - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Mixed Platforms = Release|Mixed Platforms - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Debug|Mixed Platforms.ActiveCfg = Debug|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Debug|Mixed Platforms.Build.0 = Debug|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Debug|Win32.ActiveCfg = Debug|Win32 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Debug|Win32.Build.0 = Debug|Win32 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Debug|x64.ActiveCfg = Debug|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Debug|x64.Build.0 = Debug|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Release|Mixed Platforms.ActiveCfg = Release|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Release|Mixed Platforms.Build.0 = Release|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Release|Win32.ActiveCfg = Release|Win32 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Release|Win32.Build.0 = Release|Win32 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Release|x64.ActiveCfg = Release|x64 - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62}.Release|x64.Build.0 = Release|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Debug|Mixed Platforms.ActiveCfg = Debug|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Debug|Mixed Platforms.Build.0 = Debug|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Debug|Win32.ActiveCfg = Debug|Win32 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Debug|Win32.Build.0 = Debug|Win32 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Debug|x64.ActiveCfg = Debug|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Debug|x64.Build.0 = Debug|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Release|Mixed Platforms.ActiveCfg = Release|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Release|Mixed Platforms.Build.0 = Release|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Release|Win32.ActiveCfg = Release|Win32 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Release|Win32.Build.0 = Release|Win32 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Release|x64.ActiveCfg = Release|x64 - {721AF7E3-9330-4598-AB73-5AB993FE3F9C}.Release|x64.Build.0 = Release|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Debug|Mixed Platforms.ActiveCfg = Debug|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Debug|Mixed Platforms.Build.0 = Debug|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Debug|Win32.ActiveCfg = Debug|Win32 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Debug|Win32.Build.0 = Debug|Win32 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Debug|x64.ActiveCfg = Debug|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Debug|x64.Build.0 = Debug|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Release|Mixed Platforms.ActiveCfg = Release|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Release|Mixed Platforms.Build.0 = Release|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Release|Win32.ActiveCfg = Release|Win32 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Release|Win32.Build.0 = Release|Win32 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Release|x64.ActiveCfg = Release|x64 - {ADFE2428-2595-4535-822D-4E70405F7D58}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/Pyjion/Pyjion.vcxproj b/Pyjion/Pyjion.vcxproj deleted file mode 100644 index 7365f22a1..000000000 --- a/Pyjion/Pyjion.vcxproj +++ /dev/null @@ -1,176 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - - {0B2F9AA3-F525-4042-B8C9-74F8B10F9D62} - JitTest - Pyjion - x64 - 8.1 - - - - Application - true - v140 - NotSet - - - DynamicLibrary - true - v140 - NotSet - - - Application - false - v140 - true - MultiByte - - - DynamicLibrary - false - v140 - true - MultiByte - - - - $(WindowsSDK_IncludePath);$(MSBuildProjectDirectory)\..\CoreClr\src\pal\prebuilt\inc;$(MSBuildProjectDirectory)\..\coreclr\src\inc\;$(IncludePath);$(MSBuildProjectDirectory)\..\Python\Include;$(MSBuildProjectDirectory)\..\Python\PC - - - - - - - - - - - - - - - - - - .pyd - pyjion_d - $(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(MSBuildProjectDirectory)\..\libs\$(Configuration)\$(Platform);$(MSBuildProjectDirectory)\..\Python\PCbuild\amd64 - - - .pyd - pyjion - $(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(MSBuildProjectDirectory)\..\libs\$(Configuration)\$(Platform);$(MSBuildProjectDirectory)\..\Python\PCbuild\amd64 - - - - Level3 - Disabled - true - %(PreprocessorDefinitions);_TARGET_ARM64_=1 - Cdecl - false - None - Default - false - None - - - true - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies);clrjit.lib;utilcode.lib;advapi32.lib ;OleAut32.lib;gcinfo.lib;user32.lib - - - - - Level3 - Disabled - true - %(PreprocessorDefinitions);_TARGET_AMD64_=1 - Cdecl - Sync - ProgramDatabase - Default - true - None - - - true - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies);clrjit.lib;utilcode.lib;advapi32.lib ;OleAut32.lib;gcinfo.lib;user32.lib;python36_d.lib - - - - - Level3 - MaxSpeed - true - true - true - Cdecl - - - true - true - true - - - - - Level3 - Full - true - false - true - _MBCS;%(PreprocessorDefinitions);_TARGET_AMD64_=1 - false - - - true - true - true - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies);clrjit.lib;utilcode.lib;advapi32.lib ;gcinfo.lib;user32.lib;python35.lib - - - - - - \ No newline at end of file From cf64296f8187443951f60772946a50e031348f30 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 12:05:11 +1000 Subject: [PATCH 027/273] Export module from build. Replace windows TLS API with the new PEP539 TLS API --- CMakeLists.txt | 18 ++++++++++++++++-- Pyjion/pyjit.cpp | 18 +++++++++--------- Pyjion/pyjit.h | 5 +---- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d140aacc..6aff6c4c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set (CLR_DIR CoreCLR/src/coreclr) add_compile_options(-fexceptions) add_definitions(-DUSE_STL) - +add_compile_options(-fvisibility=hidden) if(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) @@ -48,9 +48,23 @@ if (APPLE) endif() include_directories(CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/inc) -set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp) +set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp Pyjion/pyjit.cpp) add_library(pyjion SHARED ${SOURCES}) + +install(TARGETS pyjion + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +set_target_properties( + pyjion + PROPERTIES + PREFIX "" + OUTPUT_NAME "pyjit" + LINKER_LANGUAGE C +) + target_link_libraries(pyjion ${Python3_LIBRARIES}) target_link_libraries(pyjion ${CMAKE_SOURCE_DIR}/CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/${CLR_JIT_LIB}) + add_compile_options(-Wno-sign-compare) diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index c744c0cf9..615a36c69 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -121,13 +121,13 @@ PyObject* Jit_EvalHelper(void* state, PyFrameObject*frame) { Py_LeaveRecursiveCall(); frame->f_executing = 0; - return _Py_CheckFunctionResult(NULL, res, "Jit_EvalHelper"); + return _Py_CheckFunctionResult(NULL, res, (PyObject *) "Jit_EvalHelper", NULL); } -static DWORD g_extraSlot; +static Py_tss_t* g_extraSlot; extern "C" __declspec(dllexport) void JitInit() { - g_extraSlot = TlsAlloc(); + g_extraSlot = PyThread_tss_alloc(); g_jit = getJit(); @@ -236,7 +236,7 @@ PyTypeObject* GetArgType(int arg, PyObject** locals) { PyObject* Jit_EvalGeneric(PyjionJittedCode* state, PyFrameObject*frame) { auto trace = (PyjionJittedCode*)state; - return Jit_EvalHelper(trace->j_generic, frame); + return Jit_EvalHelper((void*)trace->j_generic, frame); } #define MAX_TRACE 5 @@ -346,7 +346,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { if (target != nullptr && !trace->j_failed) { if (target->addr != nullptr) { // we have a specialized function for this, just invoke it - auto res = Jit_EvalHelper(target->addr, frame); + auto res = Jit_EvalHelper((void*)target->addr, frame); return res; } @@ -403,7 +403,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { trace->j_evalfunc = Jit_EvalGeneric; } - return Jit_EvalHelper(target->addr, frame); + return Jit_EvalHelper((void*)target->addr, frame); } } @@ -453,14 +453,14 @@ __declspec(dllexport) bool jit_compile(PyCodeObject* code) { extern "C" __declspec(dllexport) PyjionJittedCode* PyJit_EnsureExtra(PyObject* codeObject) { - ssize_t index = (ssize_t)TlsGetValue(g_extraSlot); + ssize_t index = (ssize_t)PyThread_tss_get(g_extraSlot); if (index == 0) { index = _PyEval_RequestCodeExtraIndex(PyjionJitFree); if (index == -1) { return nullptr; } - TlsSetValue(g_extraSlot, (LPVOID)((index << 1) | 0x01)); + PyThread_tss_set(g_extraSlot, (LPVOID)((index << 1) | 0x01)); } else { index = index >> 1; @@ -684,7 +684,7 @@ static PyMethodDef PyjionMethods[] = { static struct PyModuleDef pyjionmodule = { PyModuleDef_HEAD_INIT, "pyjion", /* name of module */ - "Pyjion - A Just-in-Time Compiler for CPython 3.6.x", /* module documentation, may be NULL */ + "Pyjion - A Just-in-Time Compiler for CPython", /* module documentation, may be NULL */ -1, /* size of per-interpreter state of the module, or -1 if the module keeps state in global variables. */ PyjionMethods diff --git a/Pyjion/pyjit.h b/Pyjion/pyjit.h index 111d2fc96..15ca0304b 100644 --- a/Pyjion/pyjit.h +++ b/Pyjion/pyjit.h @@ -27,7 +27,7 @@ #define PYJIT_H #define FEATURE_NO_HOST -#define USE_STL + #include #include #include @@ -49,9 +49,6 @@ #include - //#define NO_TRACE - //#define TRACE_TREE - struct SpecializedTreeNode; class PyjionJittedCode; From 20e1511116dc286afb70591fc7533d2b967e4bbc Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 12:39:02 +1000 Subject: [PATCH 028/273] the eval frame API has changed, it requires interpreter/thread states. update all the functions to use the correct 3.9 APIs --- Pyjion/pyjit.cpp | 33 +++++++++++++++++---------------- Pyjion/pyjit.h | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index 615a36c69..a959f76a4 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -25,6 +25,7 @@ */ +#include #include "pyjit.h" #include "pycomp.h" @@ -103,7 +104,7 @@ PyObject* Jit_EvalHelper(void* state, PyFrameObject*frame) { PyThreadState *tstate = PyThreadState_GET(); if (tstate->use_tracing) { if (tstate->c_tracefunc != NULL) { - return _PyEval_EvalFrameDefault(frame, 0); + return _PyEval_EvalFrameDefault(tstate, frame, 0); } } @@ -245,7 +246,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { // Walk our tree of argument types to find the SpecializedTreeNode which // corresponds with our sets of arguments here. auto trace = (PyjionJittedCode*)state; - + auto tstate = PyThreadState_Get(); #ifdef TRACE_TREE // no match on specialized functions... @@ -390,7 +391,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { printf("Compilation failure #%d\r\n", ++failCount); #endif trace->j_failed = true; - return _PyEval_EvalFrameDefault(frame, 0); + return _PyEval_EvalFrameDefault(tstate, frame, 0); } // Update the jitted information for this tree node @@ -416,7 +417,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { frame ); #endif - auto res = _PyEval_EvalFrameDefault(frame, 0); + auto res = _PyEval_EvalFrameDefault(tstate, frame, 0); #ifdef DEBUG_CALL_TRACE printf("Returning default %s from %s line %d %s %p\r\n", PyUnicode_AsUTF8(frame->f_code->co_name), @@ -490,7 +491,7 @@ extern "C" __declspec(dllexport) PyjionJittedCode* PyJit_EnsureExtra(PyObject* c // and dispatch to it if it's already compiled. If it hasn't yet been compiled we'll // eventually compile it and invoke it. If it's not time to compile it yet then we'll // invoke the default evaluation function. -extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyFrameObject *f, int throwflag) { +extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyThreadState *ts,PyFrameObject *f, int throwflag) { auto err = GetLastError(); auto jitted = PyJit_EnsureExtra((PyObject*)f->f_code); @@ -536,7 +537,7 @@ extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyFrameObject *f, int ); #endif - auto res = _PyEval_EvalFrameDefault(f, throwflag); + auto res = _PyEval_EvalFrameDefault(ts, f, throwflag); #ifdef DEBUG_CALL_TRACE printf("Returning EFD %s from %s line %d %s %p\r\n", PyUnicode_AsUTF8(f->f_code->co_name), @@ -562,10 +563,13 @@ void PyjionJitFree(void* obj) { delete obj; } +static PyInterpreterState* inter(){ + return PyInterpreterState_Main(); +} + static PyObject *pyjion_enable(PyObject *self, PyObject* args) { - auto ts = PyThreadState_Get(); - auto prev = ts->interp->eval_frame; - ts->interp->eval_frame = PyJit_EvalFrame; + auto prev = _PyInterpreterState_GetEvalFrameFunc(inter()); + _PyInterpreterState_SetEvalFrameFunc(inter(), reinterpret_cast<_PyFrameEvalFunction>(PyJit_EvalFrame)); if (prev == PyJit_EvalFrame) { Py_RETURN_FALSE; } @@ -573,9 +577,8 @@ static PyObject *pyjion_enable(PyObject *self, PyObject* args) { } static PyObject *pyjion_disable(PyObject *self, PyObject* args) { - auto ts = PyThreadState_Get(); - auto prev = ts->interp->eval_frame; - ts->interp->eval_frame = _PyEval_EvalFrameDefault; + auto prev = _PyInterpreterState_GetEvalFrameFunc(inter()); + _PyInterpreterState_SetEvalFrameFunc(inter(), _PyEval_EvalFrameDefault); if (prev == PyJit_EvalFrame) { Py_RETURN_TRUE; } @@ -583,8 +586,7 @@ static PyObject *pyjion_disable(PyObject *self, PyObject* args) { } static PyObject *pyjion_status(PyObject *self, PyObject* args) { - auto ts = PyThreadState_Get(); - auto prev = ts->interp->eval_frame; + auto prev = _PyInterpreterState_GetEvalFrameFunc(inter()); if (prev == PyJit_EvalFrame) { Py_RETURN_TRUE; } @@ -695,8 +697,7 @@ PyMODINIT_FUNC PyInit_pyjion(void) // Install our frame evaluation function JitInit(); - auto ts = PyThreadState_Get(); - ts->interp->eval_frame = PyJit_EvalFrame; + _PyInterpreterState_SetEvalFrameFunc(inter(), PyJit_EvalFrame); return PyModule_Create(&pyjionmodule); } \ No newline at end of file diff --git a/Pyjion/pyjit.h b/Pyjion/pyjit.h index 15ca0304b..ff82d6a70 100644 --- a/Pyjion/pyjit.h +++ b/Pyjion/pyjit.h @@ -53,7 +53,7 @@ struct SpecializedTreeNode; class PyjionJittedCode; extern "C" __declspec(dllexport) void JitInit(); -extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyFrameObject *, int); +extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyThreadState *, PyFrameObject *, int); extern "C" __declspec(dllexport) PyjionJittedCode* PyJit_EnsureExtra(PyObject* codeObject); class PyjionJittedCode; From cf55d0115ec979dfe9bd3b4c3a8d35937f48ee5e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 13:33:01 +1000 Subject: [PATCH 029/273] Remove all the old legacy Opcodes and switch to the rich comparison operators (COMPARE_OP). TODO: Implement opcodes for in, not in, etc. special opcodes --- Pyjion/absint.cpp | 148 ++++------------------------------------------ 1 file changed, 12 insertions(+), 136 deletions(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index b6e1c6a8c..838fab66e 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -528,14 +529,12 @@ bool AbstractInterpreter::interpret() { lastState.pop(); break; case BUILD_LIST: - case BUILD_LIST_UNPACK: for (int i = 0; i < oparg; i++) { lastState.pop(); } lastState.push(&List); break; case BUILD_TUPLE: - case BUILD_TUPLE_UNPACK: { vector sources; for (int i = 0; i < oparg; i++) { @@ -561,43 +560,18 @@ bool AbstractInterpreter::interpret() { } lastState.push(&Dict); break; - case BUILD_MAP_UNPACK: - for (int i = 0; i < oparg; i++) { - lastState.pop(); - } - lastState.push(&Dict); - break; - case COMPARE_OP: - switch (oparg) { - case PyCmp_IS: - case PyCmp_IS_NOT: - case PyCmp_IN: - case PyCmp_NOT_IN: - lastState.pop(); - lastState.pop(); - lastState.push(&Bool); - break; - case PyCmp_EXC_MATCH: - // TODO: Produces an error or a bool, but no way to represent that so we're conservative - lastState.pop(); - lastState.pop(); - lastState.push(&Any); - break; - default: - { - auto two = lastState.pop_no_escape(); - auto one = lastState.pop_no_escape(); - auto binaryRes = one.Value->compare(one.Sources, oparg, two); - - auto sources = add_intermediate_source(opcodeIndex); - AbstractSource::combine( + case COMPARE_OP: { + // TODO Implement IS_OP, _CONTAINS_OP + auto two = lastState.pop_no_escape(); + auto one = lastState.pop_no_escape(); + auto binaryRes = one.Value->compare(one.Sources, oparg, two); + auto sources = add_intermediate_source(opcodeIndex); + AbstractSource::combine( AbstractSource::combine(one.Sources, two.Sources), sources - ); - auto value = AbstractValueWithSources(binaryRes, sources); - lastState.push(value); - break; - } + ); + auto value = AbstractValueWithSources(binaryRes, sources); + lastState.push(value); } break; case IMPORT_NAME: @@ -757,7 +731,6 @@ bool AbstractInterpreter::interpret() { lastState.pop(); break; case BUILD_SET: - case BUILD_SET_UNPACK: for (int i = 0; i < oparg; i++) { lastState.pop(); } @@ -821,30 +794,6 @@ bool AbstractInterpreter::interpret() { lastState.pop(); lastState.pop(); break; -// case CONTINUE_LOOP: -// if (update_start_state(lastState, oparg)) { -// queue.push_back(oparg); -// } -// // Done processing this basic block, we'll need to see a branch -// // to the following opcodes before we'll process them. -// goto next; -// case BREAK_LOOP: -// { -// auto breakTo = m_breakTo[opcodeIndex]; -// -// // BREAK_LOOP does an unwind block, which restores the -// // stack state to what it was when we entered the loop. So -// // we get the start state for where the SETUP_LOOP happened -// // here and propagate it to where we're breaking to. But we -// // need to preserve our local state as that isn't restored. -// auto startState = m_startStates[breakTo.BlockStart]; -// startState.m_locals = lastState.m_locals; -// if (update_start_state(startState, breakTo.BlockEnd)) { -// queue.push_back(breakTo.BlockEnd); -// } -// -// goto next; -// } case SETUP_FINALLY: { auto finallyState = lastState; @@ -901,9 +850,6 @@ bool AbstractInterpreter::interpret() { case SETUP_WITH: case YIELD_VALUE: return false; - case BUILD_TUPLE_UNPACK_WITH_CALL: - case BUILD_MAP_UNPACK_WITH_CALL: - return false; case BUILD_CONST_KEY_MAP: lastState.pop(); //keys for (auto i = 0; i < oparg; i++) { @@ -1339,16 +1285,10 @@ char* AbstractInterpreter::opcode_name(int opcode) { OP_TO_STR(SET_ADD) OP_TO_STR(MAP_ADD) OP_TO_STR(LOAD_CLASSDEREF) - OP_TO_STR(BUILD_LIST_UNPACK) - OP_TO_STR(BUILD_MAP_UNPACK) - OP_TO_STR(BUILD_MAP_UNPACK_WITH_CALL) - OP_TO_STR(BUILD_TUPLE_UNPACK) - OP_TO_STR(BUILD_SET_UNPACK) OP_TO_STR(SETUP_ASYNC_WITH) OP_TO_STR(FORMAT_VALUE) OP_TO_STR(BUILD_CONST_KEY_MAP) OP_TO_STR(BUILD_STRING) - OP_TO_STR(BUILD_TUPLE_UNPACK_WITH_CALL) } return "unknown"; } @@ -2168,26 +2108,14 @@ JittedCode* AbstractInterpreter::compile_worker() { build_tuple(oparg); inc_stack(); break; - case BUILD_TUPLE_UNPACK: - extend_tuple(oparg); - inc_stack(); - break; case BUILD_LIST: build_list(oparg); inc_stack(); break; - case BUILD_LIST_UNPACK: - extend_list(oparg); - inc_stack(); - break; case BUILD_MAP: build_map(oparg); inc_stack(); break; - case BUILD_MAP_UNPACK: - extend_map(oparg); - inc_stack(); - break; case STORE_SUBSCR: dec_stack(3); m_comp->emit_store_subscr(); @@ -2207,7 +2135,6 @@ JittedCode* AbstractInterpreter::compile_worker() { inc_stack(); break; case BUILD_SET: build_set(oparg); break; - case BUILD_SET_UNPACK: extend_set(oparg); inc_stack(); break; @@ -2545,18 +2472,6 @@ JittedCode* AbstractInterpreter::compile_worker() { int_error_check("import star failed"); break; case SETUP_WITH: - case BUILD_MAP_UNPACK_WITH_CALL: - /* TODO: Finish implementation - - m_comp->emit_new_dict(oparg); - auto dict = m_comp->emit_spill(); - // TODO: Null check - - for (auto i = 0; i < oparg; i++) { - } - - */ - case BUILD_TUPLE_UNPACK_WITH_CALL: return nullptr; case FORMAT_VALUE: { @@ -3324,46 +3239,7 @@ void AbstractInterpreter::for_iter(int loopIndex, int opcodeIndex, BlockInfo *lo void AbstractInterpreter::compare_op(int compareType, int& i, int opcodeIndex) { switch (compareType) { - case PyCmp_IS: - case PyCmp_IS_NOT: - // TODO: Inlining this would be nice, but then we need the dec refs, e.g.: - if (can_optimize_pop_jump(i)) { - m_comp->emit_is_push_int(compareType != PyCmp_IS); - dec_stack(); // popped 2, pushed 1 - branch(i); - } - else { - m_comp->emit_is(compareType != PyCmp_IS); - dec_stack(); - } - break; - case PyCmp_IN: - if (can_optimize_pop_jump(i)) { - m_comp->emit_in_push_int(); - dec_stack(2); - branch_or_error(i); - } - else { - m_comp->emit_in(); - dec_stack(2); - error_check("in failed"); - inc_stack(); - } - break; - case PyCmp_NOT_IN: - if (can_optimize_pop_jump(i)) { - m_comp->emit_not_in_push_int(); - dec_stack(2); - branch_or_error(i); - } - else { - m_comp->emit_not_in(); - dec_stack(2); - error_check("not in failed"); - inc_stack(); - } - break; - case PyCmp_EXC_MATCH: + case Py_EQ: if (get_extended_opcode(i + sizeof(_Py_CODEUNIT)) == POP_JUMP_IF_FALSE) { m_comp->emit_compare_exceptions_int(); dec_stack(2); From 8daa965111303283349c5fe4b6a4ec51b72e1886 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 13:57:47 +1000 Subject: [PATCH 030/273] Build the correct CPython module format and linking --- CMakeLists.txt | 7 ++----- Pyjion/pyjit.cpp | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6aff6c4c5..3277f5b3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,7 +50,7 @@ endif() include_directories(CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/inc) set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jitinit.cpp Pyjion/pycomp.cpp Pyjion/pyjit.cpp) -add_library(pyjion SHARED ${SOURCES}) +add_library(pyjion MODULE ${SOURCES}) install(TARGETS pyjion LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -60,11 +60,8 @@ set_target_properties( pyjion PROPERTIES PREFIX "" - OUTPUT_NAME "pyjit" + OUTPUT_NAME "pyjion" LINKER_LANGUAGE C ) - target_link_libraries(pyjion ${Python3_LIBRARIES}) target_link_libraries(pyjion ${CMAKE_SOURCE_DIR}/CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/${CLR_JIT_LIB}) - -add_compile_options(-Wno-sign-compare) diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index a959f76a4..a9caf8826 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -129,7 +129,6 @@ static Py_tss_t* g_extraSlot; extern "C" __declspec(dllexport) void JitInit() { g_extraSlot = PyThread_tss_alloc(); - g_jit = getJit(); g_emptyTuple = PyTuple_New(0); @@ -697,7 +696,5 @@ PyMODINIT_FUNC PyInit_pyjion(void) // Install our frame evaluation function JitInit(); - _PyInterpreterState_SetEvalFrameFunc(inter(), PyJit_EvalFrame); - return PyModule_Create(&pyjionmodule); } \ No newline at end of file From 6e61c4cb28e43e57726eb99002f850b618741070 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 16:11:41 +1000 Subject: [PATCH 031/273] Align to the correct API --- CMakeLists.txt | 8 ++++++-- Pyjion/pyjit.cpp | 27 ++++++++++++++------------- Pyjion/pyjit.h | 9 +++++---- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3277f5b3e..7c81e09de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,6 @@ set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) -include(CheckCXXCompilerFlag) - find_package (Python3 COMPONENTS Interpreter Development) include_directories(${Python3_INCLUDE_DIRS}) @@ -17,6 +15,11 @@ set (CLR_DIR CoreCLR/src/coreclr) add_compile_options(-fexceptions) add_definitions(-DUSE_STL) add_compile_options(-fvisibility=hidden) + +if (DEBUG) + add_definitions(-DDEBUG_CALL_TRACE) +endif() + if(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) @@ -63,5 +66,6 @@ set_target_properties( OUTPUT_NAME "pyjion" LINKER_LANGUAGE C ) + target_link_libraries(pyjion ${Python3_LIBRARIES}) target_link_libraries(pyjion ${CMAKE_SOURCE_DIR}/CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/${CLR_JIT_LIB}) diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index a9caf8826..12f343ace 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -29,6 +29,8 @@ #include "pyjit.h" #include "pycomp.h" +#define DEBUG_CALL_TRACE 1 + HINSTANCE g_pMSCorEE; // Tracks types for a function call. Each argument has a SpecializedTreeNode with @@ -127,10 +129,10 @@ PyObject* Jit_EvalHelper(void* state, PyFrameObject*frame) { static Py_tss_t* g_extraSlot; -extern "C" __declspec(dllexport) void JitInit() { +static void JitInit() { g_extraSlot = PyThread_tss_alloc(); + PyThread_tss_create(g_extraSlot); g_jit = getJit(); - g_emptyTuple = PyTuple_New(0); } @@ -428,7 +430,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { return res; } -__declspec(dllexport) bool jit_compile(PyCodeObject* code) { +static bool jit_compile(PyCodeObject* code) { if (strcmp(PyUnicode_AsUTF8(code->co_name), "") == 0 || strcmp(PyUnicode_AsUTF8(code->co_name), "") == 0) { return false; @@ -452,8 +454,8 @@ __declspec(dllexport) bool jit_compile(PyCodeObject* code) { #endif -extern "C" __declspec(dllexport) PyjionJittedCode* PyJit_EnsureExtra(PyObject* codeObject) { - ssize_t index = (ssize_t)PyThread_tss_get(g_extraSlot); +static PyjionJittedCode* PyJit_EnsureExtra(PyObject* codeObject) { + auto index = (ssize_t)PyThread_tss_get(g_extraSlot); if (index == 0) { index = _PyEval_RequestCodeExtraIndex(PyjionJitFree); if (index == -1) { @@ -486,17 +488,16 @@ extern "C" __declspec(dllexport) PyjionJittedCode* PyJit_EnsureExtra(PyObject* c return jitted; } -// This is our replacement evaluation function. We lookup our coorespanding jitted code +// This is our replacement evaluation function. We lookup our corresponding jitted code // and dispatch to it if it's already compiled. If it hasn't yet been compiled we'll // eventually compile it and invoke it. If it's not time to compile it yet then we'll // invoke the default evaluation function. -extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyThreadState *ts,PyFrameObject *f, int throwflag) { - auto err = GetLastError(); - +PyObject* PyJit_EvalFrame(PyThreadState *ts, PyFrameObject *f, int throwflag) { + //auto err = GetLastError(); auto jitted = PyJit_EnsureExtra((PyObject*)f->f_code); if (jitted != nullptr && !throwflag) { if (jitted->j_evalfunc != nullptr) { - SetLastError(err); + //SetLastError(err); #if DEBUG_CALL_TRACE printf("Calling %s from %s line %d %p\r\n", PyUnicode_AsUTF8(f->f_code->co_name), @@ -510,7 +511,7 @@ extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyThreadState *ts,PyF else if (!jitted->j_failed && jitted->j_run_count++ >= jitted->j_specialization_threshold) { if (jit_compile(f->f_code)) { // execute the jitted code... - SetLastError(err); + //SetLastError(err); #ifdef DEBUG_CALL_TRACE printf("Calling first %s from %s line %d %p\r\n", PyUnicode_AsUTF8(f->f_code->co_name), @@ -526,7 +527,7 @@ extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyThreadState *ts,PyF jitted->j_failed = true; } } - SetLastError(err); + //SetLastError(err); #ifdef DEBUG_CALL_TRACE printf("Falling to EFD %s from %s line %d %s %p\r\n", PyUnicode_AsUTF8(f->f_code->co_name), @@ -568,7 +569,7 @@ static PyInterpreterState* inter(){ static PyObject *pyjion_enable(PyObject *self, PyObject* args) { auto prev = _PyInterpreterState_GetEvalFrameFunc(inter()); - _PyInterpreterState_SetEvalFrameFunc(inter(), reinterpret_cast<_PyFrameEvalFunction>(PyJit_EvalFrame)); + _PyInterpreterState_SetEvalFrameFunc(inter(), PyJit_EvalFrame); if (prev == PyJit_EvalFrame) { Py_RETURN_FALSE; } diff --git a/Pyjion/pyjit.h b/Pyjion/pyjit.h index ff82d6a70..65aa82c16 100644 --- a/Pyjion/pyjit.h +++ b/Pyjion/pyjit.h @@ -52,9 +52,9 @@ struct SpecializedTreeNode; class PyjionJittedCode; -extern "C" __declspec(dllexport) void JitInit(); -extern "C" __declspec(dllexport) PyObject *PyJit_EvalFrame(PyThreadState *, PyFrameObject *, int); -extern "C" __declspec(dllexport) PyjionJittedCode* PyJit_EnsureExtra(PyObject* codeObject); +static void JitInit(); +PyObject* PyJit_EvalFrame(PyThreadState *, PyFrameObject *, int); +static PyjionJittedCode* PyJit_EnsureExtra(PyObject* codeObject); class PyjionJittedCode; typedef PyObject* (*Py_EvalFunc)(PyjionJittedCode*, struct _frame*); @@ -94,6 +94,7 @@ class PyjionJittedCode { ~PyjionJittedCode(); }; -__declspec(dllexport) bool jit_compile(PyCodeObject* code); + +static bool jit_compile(PyCodeObject* code); #endif \ No newline at end of file From f5ffd75b564ea38662c2d071f83d0c3c4e27b328 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 16:24:21 +1000 Subject: [PATCH 032/273] Add debug flags in build Make a note of an overflow error when importing the turtle package --- CMakeLists.txt | 7 ++++--- Pyjion/ilgen.h | 1 + Pyjion/pyjit.cpp | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c81e09de..d0ee77aaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,9 +16,10 @@ add_compile_options(-fexceptions) add_definitions(-DUSE_STL) add_compile_options(-fvisibility=hidden) -if (DEBUG) - add_definitions(-DDEBUG_CALL_TRACE) -endif() +IF(CMAKE_BUILD_TYPE MATCHES DEBUG) + message("Enabling very verbose messages") + add_definitions(-DDEBUG) +ENDIF(CMAKE_BUILD_TYPE MATCHES DEBUG) if(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) diff --git a/Pyjion/ilgen.h b/Pyjion/ilgen.h index e4e9b433c..af1527725 100644 --- a/Pyjion/ilgen.h +++ b/Pyjion/ilgen.h @@ -348,6 +348,7 @@ class ILGenerator { } void ld_i(void* ptr) { + // BUG : Line causes bad access Exception: EXC_BAD_ACCESS (code=2, address=0x7ffeecf15ff8) size_t value = (size_t)ptr; #ifdef _TARGET_AMD64_ if ((value & 0xFFFFFFFF) == value) { diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index 12f343ace..8b9f229fa 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -29,7 +29,9 @@ #include "pyjit.h" #include "pycomp.h" +#ifdef DEBUG #define DEBUG_CALL_TRACE 1 +#endif HINSTANCE g_pMSCorEE; From 42957adc80f001afedd8b3c454511b4f975734b8 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 17:24:16 +1000 Subject: [PATCH 033/273] Implement the new method for assigning CORJIT flags via a pointer and a callback Fix an assertion for AMD64/x86_64 requiring MSVC defined target spec --- CMakeLists.txt | 10 ++++++++++ Pyjion/ilgen.h | 2 +- Pyjion/jitinfo.h | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d0ee77aaf..37715cc22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,16 @@ if(NOT WIN32) endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif(NOT WIN32) +if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") + add_definitions(-D_TARGET_AMD64_) +elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64") + add_definitions(-D_TARGET_AMD64_) +elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64") + if (CMAKE_CL_64) + add_definitions(-D_TARGET_AMD64_) + endif() +endif() + include_directories(CoreCLR/src/coreclr/src/inc) if (WIN32) diff --git a/Pyjion/ilgen.h b/Pyjion/ilgen.h index af1527725..55a20b7a3 100644 --- a/Pyjion/ilgen.h +++ b/Pyjion/ilgen.h @@ -485,7 +485,7 @@ class ILGenerator { CorJitResult result = jit->compileMethod( /*ICorJitInfo*/jitInfo, /*CORINFO_METHOD_INFO */&methodInfo, - /*flags*/ CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_SKIP_VERIFICATION , + /*flags*/ CORJIT_FLAGS::CORJIT_FLAG_CALL_GETJITFLAGS , &nativeEntry, &nativeSizeOfCode ); diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 9a883e83e..83cc02cb0 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -1755,7 +1755,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { DWORD getJitFlags(CORJIT_FLAGS * flags, DWORD sizeInBytes) { - return 0; + flags->Add(flags->CORJIT_FLAG_SKIP_VERIFICATION); + return sizeof(CORJIT_FLAGS); } void getMethodVTableOffset(CORINFO_METHOD_HANDLE method, unsigned int *offsetOfIndirection, From 04cfc82d5ae6e7eb21d93ac5fc1399913a802d3d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 17:33:57 +1000 Subject: [PATCH 034/273] Use PyMem APIs for allocating JIT memory --- Pyjion/cee.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pyjion/cee.h b/Pyjion/cee.h index dadb4547f..6c852760f 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -28,6 +28,7 @@ #define FEATURE_NO_HOST +#include #include #include #include @@ -53,13 +54,12 @@ using namespace std; class CCorJitHost : public ICorJitHost { void * allocateMemory(size_t size) override { - // TODO - return nullptr; + return PyMem_Malloc(size); } void freeMemory(void * block) override { - // TODO.. + return PyMem_Free(block); } int getIntConfigValue(const WCHAR* name, int defaultValue) override From 41e528d7b7be1fbdde980e0fca78d8b9425492c6 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 19:13:32 +1000 Subject: [PATCH 035/273] Store settings in a Python dictionary --- CMakeLists.txt | 1 + Pyjion/cee.h | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37715cc22..c76a5b139 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ if(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) add_compile_options(-fdeclspec) + add_definitions(-DTARGET_UNIX) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-null-arithmetic) else(CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 6c852760f..89050368a 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -52,6 +52,12 @@ using namespace std; class CCorJitHost : public ICorJitHost { + protected: PyObject* settings ; + +public: CCorJitHost(){ + settings = PyDict_New(); + } + void * allocateMemory(size_t size) override { return PyMem_Malloc(size); @@ -62,18 +68,27 @@ class CCorJitHost : public ICorJitHost { return PyMem_Free(block); } + PyObject * getSettings(){ + return settings; + } + int getIntConfigValue(const WCHAR* name, int defaultValue) override { - return defaultValue; + return PyLong_AsSize_t( + PyDict_GetItemString(settings, reinterpret_cast(name))); } const WCHAR * getStringConfigValue(const WCHAR* name) override { - return nullptr; + return reinterpret_cast(PyUnicode_AsWideCharString( + PyDict_GetItemString(settings, reinterpret_cast(name)), 0)); } void freeStringConfigValue(const WCHAR* value) override { + if (value == 0) + return; + PyDict_DelItemString(settings, reinterpret_cast(value)); } void* allocateSlab(size_t size, size_t* pActualSize) override From c531455a6b85e83d6e69bb889a63b4370cc3f9c0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 20:11:12 +1000 Subject: [PATCH 036/273] Remove some more unreachable code --- Pyjion/absint.cpp | 53 ++--------------------------------------------- Pyjion/pycomp.cpp | 10 --------- 2 files changed, 2 insertions(+), 61 deletions(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 838fab66e..c71044ced 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -134,9 +134,6 @@ bool AbstractInterpreter::preprocess() { case SETUP_WITH: // not supported... return false; -// case SETUP_LOOP: -// blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), true)); -// break; case POP_BLOCK: { auto blockStart = blockStarts.back(); @@ -144,26 +141,10 @@ bool AbstractInterpreter::preprocess() { m_blockStarts[opcodeIndex] = blockStart.BlockStart; break; } -// case SETUP_EXCEPT: -// blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), false)); -// ehKind.push_back(false); -// break; case SETUP_FINALLY: blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), false)); ehKind.push_back(true); break; -// case END_FINALLY: -// m_endFinallyIsFinally[opcodeIndex] = ehKind.back(); -// ehKind.pop_back(); -// break; -// case BREAK_LOOP: -// for (auto iter = blockStarts.rbegin(); iter != blockStarts.rend(); ++iter) { -// if (iter->IsLoop) { -// m_breakTo[opcodeIndex] = *iter; -// break; -// } -// } -// break; case LOAD_GLOBAL: { auto name = PyUnicode_AsUTF8(PyTuple_GetItem(m_code->co_names, oparg)); @@ -190,7 +171,6 @@ bool AbstractInterpreter::preprocess() { case POP_JUMP_IF_FALSE: m_jumpsTo.insert(oparg); break; - } } return true; @@ -805,34 +785,6 @@ bool AbstractInterpreter::interpret() { } } break; -// case SETUP_EXCEPT: -// { -// auto ehState = lastState; -// // Except is entered with the exception object, traceback, and exception -// // type. TODO: We could type these stronger then they currently are typed -// ehState.push(&Any); -// ehState.push(&Any); -// ehState.push(&Any); -// if (update_start_state(ehState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { -// queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); -// } -// } -// break; -// case END_FINALLY: -// { -// bool isFinally = m_endFinallyIsFinally[opcodeIndex]; -// if (isFinally) { -// // single value indicating whether we're unwinding or -// // completed normally. -// lastState.pop(); -// } -// else { -// lastState.pop(); -// lastState.pop(); -// lastState.pop(); -// } -// break; -// } case FORMAT_VALUE: if ((oparg & FVS_MASK) == FVS_HAVE_SPEC) { // format spec @@ -858,13 +810,12 @@ bool AbstractInterpreter::interpret() { lastState.push(&Dict); return false; default: -#ifdef _DEBUG printf("Unknown unsupported opcode: %s", opcode_name(opcode)); -#endif + PyErr_Format(PyExc_ValueError, + "Unknown unsupported opcode: %s", opcode_name(opcode)); return false; } update_start_state(lastState, curByte + sizeof(_Py_CODEUNIT)); - } next:; } while (queue.size() != 0); diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 153c99607..564080984 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -26,7 +26,6 @@ #include #include -#include #include "pycomp.h" @@ -41,15 +40,6 @@ void CeeInit() { #endif } -class InitHolder { -public: - InitHolder() { - CeeInit(); - } -}; - -InitHolder g_initHolder; - Module g_module; ICorJitCompiler* g_jit; From 2e25c07d9609bed4ba9f3cdc5b4f0fb406e5c1c7 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 29 Sep 2020 21:07:55 +1000 Subject: [PATCH 037/273] Undo some removed code. --- Pyjion/pycomp.cpp | 24 +++++++++++++++++------- Pyjion/pyjit.cpp | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 564080984..8ac42a054 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -40,6 +40,15 @@ void CeeInit() { #endif } +class InitHolder { +public: + InitHolder() { + CeeInit(); + } +}; + +InitHolder g_initHolder; + Module g_module; ICorJitCompiler* g_jit; @@ -615,13 +624,14 @@ void PythonCompiler::emit_call_with_tuple() { } bool PythonCompiler::emit_kwcall(size_t argCnt) { - /* - switch (argCnt) { - case 1: m_il.emit_call(METHOD_KWCALL1_TOKEN); return true; - case 2: m_il.emit_call(METHOD_KWCALL2_TOKEN); return true; - case 3: m_il.emit_call(METHOD_KWCALL3_TOKEN); return true; - case 4: m_il.emit_call(METHOD_KWCALL4_TOKEN); return true; - }*/ +// switch (argCnt) { +// case 0: m_il.emit_call(METHOD_KWCALL0_TOKEN); return true; +// case 1: m_il.emit_call(METHOD_KWCALL1_TOKEN); return true; +// case 2: m_il.emit_call(METHOD_KWCALL2_TOKEN); return true; +// case 3: m_il.emit_call(METHOD_KWCALL3_TOKEN); return true; +// case 4: m_il.emit_call(METHOD_KWCALL4_TOKEN); return true; +// default: m_il.emit_call(METHOD_KWCALLN_TOKEN); return true; +// } return false; } diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index 8b9f229fa..c702116e2 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -29,9 +29,9 @@ #include "pyjit.h" #include "pycomp.h" -#ifdef DEBUG + #define DEBUG_CALL_TRACE 1 -#endif +#define DEBUG_TRACE 1 HINSTANCE g_pMSCorEE; From df6676aa8345787adf6d9031a05514a5f87f284b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 13:12:44 +1000 Subject: [PATCH 038/273] Update the list of opcodes and gracefully handle missing opcodes instead of corrupting the IL. Create a settings map for the CEE class and enable some debug flags. Implement more of the ilgen methods to describe class methods and static methods. --- CMakeLists.txt | 21 +++- Pyjion/absint.cpp | 253 +++++++++++++++++++++++---------------------- Pyjion/cee.h | 34 +++--- Pyjion/codemodel.h | 17 ++- Pyjion/ilgen.h | 28 +++-- Pyjion/jitinfo.h | 172 +++++++++++++----------------- 6 files changed, 272 insertions(+), 253 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c76a5b139..1d9290b44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,16 +34,26 @@ if(NOT WIN32) endif(NOT WIN32) if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") - add_definitions(-D_TARGET_AMD64_) + set(IS_64 1) elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64") - add_definitions(-D_TARGET_AMD64_) + set(IS_64 1) elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64") if (CMAKE_CL_64) - add_definitions(-D_TARGET_AMD64_) + set(IS_64 1) endif() endif() -include_directories(CoreCLR/src/coreclr/src/inc) +if (IS_64) + add_definitions(-D_TARGET_AMD64_) + add_definitions(-DTARGET_AMD64) + if (NOT WIN32) + add_definitions(-DUNIX_AMD64_ABI) # assume 64-bit + endif() +else() + add_definitions(-DTARGET_X86) +endif() + +include_directories(CoreCLR/src/coreclr/src/inc CoreCLR/src/coreclr/src/jit) if (WIN32) set(CLR_OS_BUILD WIN.x64.Debug) @@ -51,7 +61,7 @@ if (WIN32) endif() if (LINUX) - set(CLR_OS_BUILD OSX.x64.Debug) + set(CLR_OS_BUILD OSX.x64.Debug) # todo - find out what this is called set(CLR_JIT_LIB "libclrjit.so") endif() @@ -60,6 +70,7 @@ if (APPLE) include_directories(Pyjion/compat) set(CLR_CMAKE_HOST_UNIX 1) set(CLR_JIT_LIB "libclrjit.dylib") + add_definitions(-D_XOPEN_SOURCE) endif() include_directories(CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/inc) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index c71044ced..a1de5df1c 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -66,6 +66,7 @@ bool AbstractInterpreter::preprocess() { if (m_code->co_flags & (CO_COROUTINE | CO_GENERATOR)) { // Don't compile co-routines or generators. We can't rely on // detecting yields because they could be optimized out. + printf("Skipping function because it contains a coroutine/generator."); return false; } for (int i = 0; i < m_code->co_argcount; i++) { @@ -79,21 +80,9 @@ bool AbstractInterpreter::preprocess() { for (size_t curByte = 0; curByte < m_size; curByte += sizeof(_Py_CODEUNIT)) { auto opcodeIndex = curByte; auto byte = GET_OPCODE(curByte); - oparg = GET_OPARG(curByte); - - // POP_BLOCK can be removed because it is unreachable, e.g. if you do: - // def f(): - // x = 1 - // y = 0 - // try: - // return x / y - // except: - // return 42 - // In Python 3.5 you would always have POP_BLOCK's, but in 3.6 the POP_BLOCK - // that should come after return x / y is gone. So we need to check - // the opcode offsets and apply the block pops here. + oparg = GET_OPARG(curByte); processOpCode: - while (blockStarts.size() != 0 && + while (!blockStarts.empty() && opcodeIndex >= blockStarts[blockStarts.size() - 1].BlockEnd) { auto blockStart = blockStarts.back(); blockStarts.pop_back(); @@ -131,16 +120,6 @@ bool AbstractInterpreter::preprocess() { m_assignmentState[oparg] = false; } break; - case SETUP_WITH: - // not supported... - return false; - case POP_BLOCK: - { - auto blockStart = blockStarts.back(); - blockStarts.pop_back(); - m_blockStarts[opcodeIndex] = blockStart.BlockStart; - break; - } case SETUP_FINALLY: blockStarts.push_back(AbsIntBlockInfo(opcodeIndex, oparg + curByte + sizeof(_Py_CODEUNIT), false)); ehKind.push_back(true); @@ -214,6 +193,7 @@ void AbstractInterpreter::init_starting_state() { bool AbstractInterpreter::interpret() { if (!preprocess()) { + printf("Failed to preprocess"); return false; } @@ -800,7 +780,14 @@ bool AbstractInterpreter::interpret() { lastState.push(&String); break; case SETUP_WITH: + lastState.push(&Any); + if (update_start_state(lastState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { + queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); + } + lastState.push(&Any); + break; case YIELD_VALUE: + printf("Unsupported situation.."); return false; case BUILD_CONST_KEY_MAP: lastState.pop(); //keys @@ -1139,107 +1126,123 @@ char* AbstractInterpreter::opcode_name(int opcode) { #define OP_TO_STR(x) case x: return #x; switch (opcode) { OP_TO_STR(POP_TOP) - OP_TO_STR(ROT_TWO) - OP_TO_STR(ROT_THREE) - OP_TO_STR(DUP_TOP) - OP_TO_STR(DUP_TOP_TWO) - OP_TO_STR(NOP) - OP_TO_STR(UNARY_POSITIVE) - OP_TO_STR(UNARY_NEGATIVE) - OP_TO_STR(UNARY_NOT) - OP_TO_STR(UNARY_INVERT) - OP_TO_STR(BINARY_MATRIX_MULTIPLY) - OP_TO_STR(INPLACE_MATRIX_MULTIPLY) - OP_TO_STR(BINARY_POWER) - OP_TO_STR(BINARY_MULTIPLY) - OP_TO_STR(BINARY_MODULO) - OP_TO_STR(BINARY_ADD) - OP_TO_STR(BINARY_SUBTRACT) - OP_TO_STR(BINARY_SUBSCR) - OP_TO_STR(BINARY_FLOOR_DIVIDE) - OP_TO_STR(BINARY_TRUE_DIVIDE) - OP_TO_STR(INPLACE_FLOOR_DIVIDE) - OP_TO_STR(INPLACE_TRUE_DIVIDE) - OP_TO_STR(GET_AITER) - OP_TO_STR(GET_ANEXT) - OP_TO_STR(BEFORE_ASYNC_WITH) - OP_TO_STR(INPLACE_ADD) - OP_TO_STR(INPLACE_SUBTRACT) - OP_TO_STR(INPLACE_MULTIPLY) - OP_TO_STR(INPLACE_MODULO) - OP_TO_STR(STORE_SUBSCR) - OP_TO_STR(DELETE_SUBSCR) - OP_TO_STR(BINARY_LSHIFT) - OP_TO_STR(BINARY_RSHIFT) - OP_TO_STR(BINARY_AND) - OP_TO_STR(BINARY_XOR) - OP_TO_STR(BINARY_OR) - OP_TO_STR(INPLACE_POWER) - OP_TO_STR(GET_ITER) - OP_TO_STR(PRINT_EXPR) - OP_TO_STR(LOAD_BUILD_CLASS) - OP_TO_STR(YIELD_FROM) - OP_TO_STR(GET_AWAITABLE) - OP_TO_STR(INPLACE_LSHIFT) - OP_TO_STR(INPLACE_RSHIFT) - OP_TO_STR(INPLACE_AND) - OP_TO_STR(INPLACE_XOR) - OP_TO_STR(INPLACE_OR) - OP_TO_STR(RETURN_VALUE) - OP_TO_STR(IMPORT_STAR) - OP_TO_STR(YIELD_VALUE) - OP_TO_STR(POP_BLOCK) - OP_TO_STR(POP_EXCEPT) - OP_TO_STR(STORE_NAME) - OP_TO_STR(DELETE_NAME) - OP_TO_STR(UNPACK_SEQUENCE) - OP_TO_STR(FOR_ITER) - OP_TO_STR(UNPACK_EX) - OP_TO_STR(STORE_ATTR) - OP_TO_STR(DELETE_ATTR) - OP_TO_STR(STORE_GLOBAL) - OP_TO_STR(DELETE_GLOBAL) - OP_TO_STR(LOAD_CONST) - OP_TO_STR(LOAD_NAME) - OP_TO_STR(BUILD_TUPLE) - OP_TO_STR(BUILD_LIST) - OP_TO_STR(BUILD_SET) - OP_TO_STR(BUILD_MAP) - OP_TO_STR(LOAD_ATTR) - OP_TO_STR(COMPARE_OP) - OP_TO_STR(IMPORT_NAME) - OP_TO_STR(IMPORT_FROM) - OP_TO_STR(JUMP_FORWARD) - OP_TO_STR(JUMP_IF_FALSE_OR_POP) - OP_TO_STR(JUMP_IF_TRUE_OR_POP) - OP_TO_STR(JUMP_ABSOLUTE) - OP_TO_STR(POP_JUMP_IF_FALSE) - OP_TO_STR(POP_JUMP_IF_TRUE) - OP_TO_STR(LOAD_GLOBAL) - OP_TO_STR(SETUP_FINALLY) - OP_TO_STR(LOAD_FAST) - OP_TO_STR(STORE_FAST) - OP_TO_STR(DELETE_FAST) - OP_TO_STR(RAISE_VARARGS) - OP_TO_STR(CALL_FUNCTION) - OP_TO_STR(MAKE_FUNCTION) - OP_TO_STR(BUILD_SLICE) - OP_TO_STR(LOAD_CLOSURE) - OP_TO_STR(LOAD_DEREF) - OP_TO_STR(STORE_DEREF) - OP_TO_STR(DELETE_DEREF) - OP_TO_STR(CALL_FUNCTION_EX) - OP_TO_STR(CALL_FUNCTION_KW) - OP_TO_STR(SETUP_WITH) - OP_TO_STR(EXTENDED_ARG) - OP_TO_STR(LIST_APPEND) - OP_TO_STR(SET_ADD) - OP_TO_STR(MAP_ADD) - OP_TO_STR(LOAD_CLASSDEREF) - OP_TO_STR(SETUP_ASYNC_WITH) - OP_TO_STR(FORMAT_VALUE) - OP_TO_STR(BUILD_CONST_KEY_MAP) - OP_TO_STR(BUILD_STRING) + OP_TO_STR(ROT_TWO) + OP_TO_STR(ROT_THREE) + OP_TO_STR(DUP_TOP) + OP_TO_STR(DUP_TOP_TWO) + OP_TO_STR(ROT_FOUR) + OP_TO_STR(NOP) + OP_TO_STR(UNARY_POSITIVE) + OP_TO_STR(UNARY_NEGATIVE) + OP_TO_STR(UNARY_NOT) + OP_TO_STR(UNARY_INVERT) + OP_TO_STR(BINARY_MATRIX_MULTIPLY) + OP_TO_STR(INPLACE_MATRIX_MULTIPLY) + OP_TO_STR(BINARY_POWER) + OP_TO_STR(BINARY_MULTIPLY) + OP_TO_STR(BINARY_MODULO) + OP_TO_STR(BINARY_ADD) + OP_TO_STR(BINARY_SUBTRACT) + OP_TO_STR(BINARY_SUBSCR) + OP_TO_STR(BINARY_FLOOR_DIVIDE) + OP_TO_STR(BINARY_TRUE_DIVIDE) + OP_TO_STR(INPLACE_FLOOR_DIVIDE) + OP_TO_STR(INPLACE_TRUE_DIVIDE) + OP_TO_STR(RERAISE) + OP_TO_STR(WITH_EXCEPT_START) + OP_TO_STR(GET_AITER) + OP_TO_STR(GET_ANEXT) + OP_TO_STR(BEFORE_ASYNC_WITH) + OP_TO_STR(END_ASYNC_FOR) + OP_TO_STR(INPLACE_ADD) + OP_TO_STR(INPLACE_SUBTRACT) + OP_TO_STR(INPLACE_MULTIPLY) + OP_TO_STR(INPLACE_MODULO) + OP_TO_STR(STORE_SUBSCR) + OP_TO_STR(DELETE_SUBSCR) + OP_TO_STR(BINARY_LSHIFT) + OP_TO_STR(BINARY_RSHIFT) + OP_TO_STR(BINARY_AND) + OP_TO_STR(BINARY_XOR) + OP_TO_STR(BINARY_OR) + OP_TO_STR(INPLACE_POWER) + OP_TO_STR(GET_ITER) + OP_TO_STR(GET_YIELD_FROM_ITER) + OP_TO_STR(PRINT_EXPR) + OP_TO_STR(LOAD_BUILD_CLASS) + OP_TO_STR(YIELD_FROM) + OP_TO_STR(GET_AWAITABLE) + OP_TO_STR(INPLACE_LSHIFT) + OP_TO_STR(INPLACE_RSHIFT) + OP_TO_STR(INPLACE_AND) + OP_TO_STR(INPLACE_XOR) + OP_TO_STR(INPLACE_OR) + OP_TO_STR(LIST_TO_TUPLE) + OP_TO_STR(RETURN_VALUE) + OP_TO_STR(IMPORT_STAR) + OP_TO_STR(SETUP_ANNOTATIONS) + OP_TO_STR(YIELD_VALUE) + OP_TO_STR(POP_BLOCK) + OP_TO_STR(POP_EXCEPT) + OP_TO_STR(STORE_NAME) + OP_TO_STR(DELETE_NAME) + OP_TO_STR(UNPACK_SEQUENCE) + OP_TO_STR(FOR_ITER) + OP_TO_STR(UNPACK_EX) + OP_TO_STR(STORE_ATTR) + OP_TO_STR(DELETE_ATTR) + OP_TO_STR(STORE_GLOBAL) + OP_TO_STR(DELETE_GLOBAL) + OP_TO_STR(LOAD_CONST) + OP_TO_STR(LOAD_NAME) + OP_TO_STR(BUILD_TUPLE) + OP_TO_STR(BUILD_LIST) + OP_TO_STR(BUILD_SET) + OP_TO_STR(BUILD_MAP) + OP_TO_STR(LOAD_ATTR) + OP_TO_STR(COMPARE_OP) + OP_TO_STR(IMPORT_NAME) + OP_TO_STR(IMPORT_FROM) + OP_TO_STR(JUMP_FORWARD) + OP_TO_STR(JUMP_IF_FALSE_OR_POP) + OP_TO_STR(JUMP_IF_TRUE_OR_POP) + OP_TO_STR(JUMP_ABSOLUTE) + OP_TO_STR(POP_JUMP_IF_FALSE) + OP_TO_STR(POP_JUMP_IF_TRUE) + OP_TO_STR(LOAD_GLOBAL) + OP_TO_STR(IS_OP) + OP_TO_STR(CONTAINS_OP) + OP_TO_STR(JUMP_IF_NOT_EXC_MATCH) + OP_TO_STR(SETUP_FINALLY) + OP_TO_STR(LOAD_FAST) + OP_TO_STR(STORE_FAST) + OP_TO_STR(DELETE_FAST) + OP_TO_STR(RAISE_VARARGS) + OP_TO_STR(CALL_FUNCTION) + OP_TO_STR(MAKE_FUNCTION) + OP_TO_STR(BUILD_SLICE) + OP_TO_STR(LOAD_CLOSURE) + OP_TO_STR(LOAD_DEREF) + OP_TO_STR(STORE_DEREF) + OP_TO_STR(DELETE_DEREF) + OP_TO_STR(CALL_FUNCTION_EX) + OP_TO_STR(CALL_FUNCTION_KW) + OP_TO_STR(SETUP_WITH) + OP_TO_STR(EXTENDED_ARG) + OP_TO_STR(LIST_APPEND) + OP_TO_STR(SET_ADD) + OP_TO_STR(MAP_ADD) + OP_TO_STR(LOAD_CLASSDEREF) + OP_TO_STR(SETUP_ASYNC_WITH) + OP_TO_STR(FORMAT_VALUE) + OP_TO_STR(BUILD_CONST_KEY_MAP) + OP_TO_STR(BUILD_STRING) + OP_TO_STR(LOAD_METHOD) + OP_TO_STR(CALL_METHOD) + OP_TO_STR(LIST_EXTEND) + OP_TO_STR(SET_UPDATE) + OP_TO_STR(DICT_MERGE) + OP_TO_STR(DICT_UPDATE) } return "unknown"; } diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 89050368a..5639f5d62 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -46,16 +46,28 @@ #include #include +#include #include "openum.h" using namespace std; class CCorJitHost : public ICorJitHost { - protected: PyObject* settings ; +protected: + map intSettings; + map strSettings; public: CCorJitHost(){ - settings = PyDict_New(); + intSettings = map(); + strSettings = map(); + +#if DEBUG + intSettings = { + "JitLsraStats": 1, + "DumpJittedMethods": 1, + "JitDumpToDebugger": 1 + }; +#endif } void * allocateMemory(size_t size) override @@ -68,27 +80,23 @@ public: CCorJitHost(){ return PyMem_Free(block); } - PyObject * getSettings(){ - return settings; - } - int getIntConfigValue(const WCHAR* name, int defaultValue) override { - return PyLong_AsSize_t( - PyDict_GetItemString(settings, reinterpret_cast(name))); + if (intSettings.find(name) != intSettings.end()) + return intSettings[name]; + return defaultValue; } const WCHAR * getStringConfigValue(const WCHAR* name) override { - return reinterpret_cast(PyUnicode_AsWideCharString( - PyDict_GetItemString(settings, reinterpret_cast(name)), 0)); + if (strSettings.find(name) != strSettings.end()) + return strSettings[name]; + return NULL; } void freeStringConfigValue(const WCHAR* value) override { - if (value == 0) - return; - PyDict_DelItemString(settings, reinterpret_cast(value)); + // noop. } void* allocateSlab(size_t size, size_t* pActualSize) override diff --git a/Pyjion/codemodel.h b/Pyjion/codemodel.h index 791a0a00d..c90f4de79 100644 --- a/Pyjion/codemodel.h +++ b/Pyjion/codemodel.h @@ -98,11 +98,12 @@ class BaseMethod { virtual void get_call_info(CORINFO_CALL_INFO *pResult) = 0; virtual DWORD get_method_attrs() { - return CORINFO_FLG_STATIC | CORINFO_FLG_NATIVE; + return CORINFO_FLG_STATIC | CORINFO_FLG_NATIVE ; } virtual void findSig(CORINFO_SIG_INFO *sig) = 0; virtual void* get_addr() = 0; virtual void getFunctionEntryPoint(CORINFO_CONST_LOOKUP * pResult) = 0; + virtual CORINFO_CLASS_HANDLE getClass() = 0; }; class Method : public BaseMethod { @@ -146,6 +147,11 @@ class Method : public BaseMethod { pResult->accessType = IAT_PVALUE; pResult->addr = &m_addr; } + + virtual CORINFO_CLASS_HANDLE getClass() { + // TODO: Implement + return NULL; + } }; class IndirectDispatchMethod : public BaseMethod { @@ -174,7 +180,14 @@ class IndirectDispatchMethod : public BaseMethod { pResult->accessType = IAT_PVALUE; pResult->addr = &m_addr; } - + virtual CORINFO_CLASS_HANDLE getClass() { + CORINFO_CLASS_HANDLE result = NULL; + // TODO : Infer type from method. +// MethodDesc* method = GetMethod(methodHnd); +// TypeHandle th = TypeHandle(method->GetMethodTable()); +// result = CORINFO_CLASS_HANDLE(th.AsPtr()); + return result; + } }; // Not currently used... diff --git a/Pyjion/ilgen.h b/Pyjion/ilgen.h index 55a20b7a3..2015a50de 100644 --- a/Pyjion/ilgen.h +++ b/Pyjion/ilgen.h @@ -44,6 +44,7 @@ #include #include +#include #include #include @@ -348,7 +349,6 @@ class ILGenerator { } void ld_i(void* ptr) { - // BUG : Line causes bad access Exception: EXC_BAD_ACCESS (code=2, address=0x7ffeecf15ff8) size_t value = (size_t)ptr; #ifdef _TARGET_AMD64_ if ((value & 0xFFFFFFFF) == value) { @@ -482,15 +482,25 @@ class ILGenerator { ULONG nativeSizeOfCode; auto res = Method(m_module, m_retType, m_params, nullptr); CORINFO_METHOD_INFO methodInfo = to_method(&res, stackSize); + CorJitResult result = jit->compileMethod( - /*ICorJitInfo*/jitInfo, - /*CORINFO_METHOD_INFO */&methodInfo, - /*flags*/ CORJIT_FLAGS::CORJIT_FLAG_CALL_GETJITFLAGS , - &nativeEntry, - &nativeSizeOfCode - ); - if (result == CORJIT_OK) { - res.m_addr = nativeEntry; + jitInfo, + &methodInfo, + CORJIT_FLAGS::CORJIT_FLAG_CALL_GETJITFLAGS, + &nativeEntry, + &nativeSizeOfCode + ); + switch (result){ + case CORJIT_OK: + res.m_addr = nativeEntry; + break; + case CORJIT_BADCODE: + case CORJIT_OUTOFMEM: + case CORJIT_INTERNALERROR: + case CORJIT_SKIPPED: + case CORJIT_RECOVERABLEERROR: + printf("Got failure code from JIT."); + break; } return res; } diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 83cc02cb0..fb0592c38 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -233,26 +233,26 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual DWORD getThreadTLSIndex( void **ppIndirection = NULL ) { - printf("getThreadTLSIndex\r\n"); + printf("getThreadTLSIndex not implemented\r\n"); return 0; } virtual const void * getInlinedCallFrameVptr( void **ppIndirection = NULL ) { - printf("getInlinedCallFrameVptr\r\n"); + printf("getInlinedCallFrameVptr not implemented\r\n"); return NULL; } virtual LONG * getAddrOfCaptureThreadGlobal( void **ppIndirection = NULL ) { - printf("getAddrOfCaptureThreadGlobal\r\n"); + printf("getAddrOfCaptureThreadGlobal not implemented\r\n"); return NULL; } virtual SIZE_T* getAddrModuleDomainID(CORINFO_MODULE_HANDLE module) { - printf("getAddrModuleDomainID\r\n"); + printf("getAddrModuleDomainID not implemented\r\n"); return 0; } @@ -273,7 +273,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual void getFunctionFixedEntryPoint( CORINFO_METHOD_HANDLE ftn, CORINFO_CONST_LOOKUP * pResult) { - printf("getFunctionFixedEntryPoint\r\n"); + printf("getFunctionFixedEntryPoint not implemented\r\n"); } // get the synchronization handle that is passed to monXstatic function @@ -281,7 +281,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE ftn, void **ppIndirection = NULL ) { - printf("getMethodSync\r\n"); + printf("getMethodSync not implemented\r\n"); return nullptr; } @@ -312,21 +312,21 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_MODULE_HANDLE handle, void **ppIndirection = NULL ) { - printf("embedModuleHandle\r\n"); return nullptr; + printf("embedModuleHandle not implemented\r\n"); return nullptr; } virtual CORINFO_CLASS_HANDLE embedClassHandle( CORINFO_CLASS_HANDLE handle, void **ppIndirection = NULL ) { - printf("embedClassHandle\r\n"); return nullptr; + printf("embedClassHandle not implemented\r\n"); return nullptr; } virtual CORINFO_METHOD_HANDLE embedMethodHandle( CORINFO_METHOD_HANDLE handle, void **ppIndirection = NULL ) { - printf("embedMethodHandle\r\n"); + printf("embedMethodHandle not implemented\r\n"); *ppIndirection = NULL; return handle; } @@ -335,7 +335,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_FIELD_HANDLE handle, void **ppIndirection = NULL ) { - printf("embedFieldHandle\r\n"); return nullptr; + printf("embedFieldHandle not implemented\r\n"); return nullptr; } // Given a module scope (module), a method handle (context) and @@ -349,7 +349,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_RESOLVED_TOKEN * pResolvedToken, BOOL fEmbedParent, // TRUE - embeds parent type handle of the field/method handle CORINFO_GENERICHANDLE_RESULT * pResult) { - printf("embedGenericHandle\r\n"); + printf("embedGenericHandle not implemented\r\n"); } // Return information used to locate the exact enclosing type of the current method. @@ -371,7 +371,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE method, void **ppIndirection = NULL ) { - printf("getPInvokeUnmanagedTarget\r\n"); + printf("getPInvokeUnmanagedTarget not implemented\r\n"); return nullptr; } @@ -380,7 +380,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE method, void **ppIndirection = NULL ) { - printf("getAddressOfPInvokeFixup\r\n"); + printf("getAddressOfPInvokeFixup not implemented\r\n"); return nullptr; } @@ -390,7 +390,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_SIG_INFO* szMetaSig, void ** ppIndirection = NULL ) { - printf("GetCookieForPInvokeCalliSig\r\n"); + printf("GetCookieForPInvokeCalliSig not implemented\r\n"); return nullptr; } @@ -409,8 +409,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE method, CORINFO_JUST_MY_CODE_HANDLE**ppIndirection = NULL ) { - printf("getJustMyCodeHandle\r\n"); - return nullptr; + CORINFO_JUST_MY_CODE_HANDLE result = NULL; + if (ppIndirection) + *ppIndirection = NULL; + DWORD * pFlagAddr = NULL; + result = (CORINFO_JUST_MY_CODE_HANDLE) pFlagAddr; + return result; } // Gets a method handle that can be used to correlate profiling data. @@ -438,7 +442,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { //Jit info CORINFO_CALLINFO_FLAGS flags, - //out params + //out params (OUT) CORINFO_CALL_INFO *pResult ) { auto method = (BaseMethod*)pResolvedToken->hMethod; @@ -450,19 +454,18 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { pResult->sig.retTypeClass = nullptr; pResult->verSig = pResult->sig; pResult->accessAllowed = CORINFO_ACCESS_ALLOWED; - //printf("getCallInfo\r\n"); } virtual BOOL canAccessFamily(CORINFO_METHOD_HANDLE hCaller, CORINFO_CLASS_HANDLE hInstanceType) { - printf("canAccessFamily\r\n"); + printf("canAccessFamily \r\n"); return FALSE; } // Returns TRUE if the Class Domain ID is the RID of the class (currently true for every class // except reflection emitted classes and generics) virtual BOOL isRIDClassDomainID(CORINFO_CLASS_HANDLE cls) { - printf("isRIDClassDomainID\r\n"); + printf("isRIDClassDomainID \r\n"); return FALSE; } @@ -471,7 +474,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_CLASS_HANDLE cls, void **ppIndirection = NULL ) { - printf("getClassDomainID\r\n"); + printf("getClassDomainID not implemented\r\n"); return 0; } @@ -481,7 +484,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_FIELD_HANDLE field, void **ppIndirection = NULL ) { - printf("getFieldAddress\r\n"); + printf("getFieldAddress not implemented\r\n"); return nullptr; } @@ -490,7 +493,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_SIG_INFO *pSig, void **ppIndirection = NULL ) { - printf("getVarArgsHandle\r\n"); + printf("getVarArgsHandle not implemented\r\n"); return nullptr; } @@ -524,7 +527,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual DWORD getMethodAttribs( CORINFO_METHOD_HANDLE ftn /* IN */ ) { - //printf("getMethodAttribs\r\n"); auto method = (BaseMethod*)ftn; return method->get_method_attrs(); } @@ -534,7 +536,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE ftn, /* IN */ CorInfoMethodRuntimeFlags attribs /* IN */ ) { - //printf("setMethodAttribs\r\n"); + printf("setMethodAttribs not implemented\r\n"); } // Given a method descriptor ftnHnd, extract signature information into sigInfo @@ -564,7 +566,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE ftn, /* IN */ CORINFO_METHOD_INFO* info /* OUT */ ) { - printf("getMethodInfo\r\n"); return false; + printf("getMethodInfo not implemented\r\n"); return false; } // Decides if you have any limitations for inlining. If everything's OK, it will return @@ -632,15 +634,15 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual CORINFO_CLASS_HANDLE getMethodClass( CORINFO_METHOD_HANDLE method ) { - //printf("getMethodClass\r\n"); - return nullptr; + auto meth = (BaseMethod*)method; + return meth->getClass(); } // return module it belongs to virtual CORINFO_MODULE_HANDLE getMethodModule( CORINFO_METHOD_HANDLE method ) { - printf("getMethodModule\r\n"); return nullptr; + printf("getMethodModule not implemented\r\n"); return nullptr; } // This function returns the offset of the specified method in the @@ -903,12 +905,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_CONTEXT_HANDLE context, /* IN */ CORINFO_SIG_INFO *sig /* OUT */ ) { - //printf("findCallSiteSig\r\n"); + printf("findCallSiteSig\r\n"); } virtual CORINFO_CLASS_HANDLE getTokenTypeAsHandle( CORINFO_RESOLVED_TOKEN * pResolvedToken /* IN */) { - printf("getTokenTypeAsHandle\r\n"); + printf("getTokenTypeAsHandle not implemented\r\n"); return nullptr; } @@ -972,9 +974,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual const char* getClassName( CORINFO_CLASS_HANDLE cls ) { - //printf("getClassName\r\n"); - return NULL; - + return "classname"; } @@ -1008,8 +1008,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual DWORD getClassAttribs( CORINFO_CLASS_HANDLE cls ) { - //printf("getClassAttribs\r\n"); - return 0; + // TODO : Load from a base class and establish correct attribs. + return CORINFO_FLG_VALUECLASS; } // Returns "TRUE" iff "cls" is a struct type such that return buffers used for returning a value @@ -1026,7 +1026,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual CORINFO_MODULE_HANDLE getClassModule( CORINFO_CLASS_HANDLE cls ) { - printf("getClassModule\r\n"); + printf("getClassModule not implemented\r\n"); return nullptr; } @@ -1034,7 +1034,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual CORINFO_ASSEMBLY_HANDLE getModuleAssembly( CORINFO_MODULE_HANDLE mod ) { - printf("getModuleAssembly\r\n"); + printf("getModuleAssembly not implemented\r\n"); return nullptr; } @@ -1042,7 +1042,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual const char* getAssemblyName( CORINFO_ASSEMBLY_HANDLE assem ) { - printf("getAssemblyName\r\n"); + printf("getAssemblyName not implemented\r\n"); return nullptr; } @@ -1060,7 +1060,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_MODULE_HANDLE *pModule, void **ppIndirection ) { - printf("getClassModuleIdForStatics\r\n"); + printf("getClassModuleIdForStatics not implemented\r\n"); return 0; } @@ -1068,7 +1068,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual unsigned getClassSize( CORINFO_CLASS_HANDLE cls ) { - printf("getClassSize\r\n"); + printf("getClassSize not implemented\r\n"); return 0; } @@ -1110,7 +1110,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { INT num ) { printf("getFieldInClass\r\n"); - return nullptr; + return NULL; } virtual BOOL checkMethodModifier( @@ -1170,8 +1170,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual CORINFO_CLASS_HANDLE getTypeForBox( CORINFO_CLASS_HANDLE cls ) { - printf("getTypeForBox\r\n"); - return nullptr; + printf("getTypeForBox not implemented\r\n"); + return NULL; } // returns the correct box helper for a particular class. Note @@ -1216,7 +1216,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual const char* getHelperName( CorInfoHelpFunc ) { - printf("getHelperName\r\n"); + printf("getHelperName not implemented\r\n"); return nullptr; } @@ -1292,7 +1292,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2 ) { - printf("mergeClasses\r\n"); + printf("mergeClasses not implemented\r\n"); return nullptr; } @@ -1302,8 +1302,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual CORINFO_CLASS_HANDLE getParentType( CORINFO_CLASS_HANDLE cls ) { - printf("getParentType\r\n"); - return NULL; + printf("getParentType not implemented\r\n"); + return nullptr; } // Returns the CorInfoType of the "child type". If the child type is @@ -1314,7 +1314,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_CLASS_HANDLE clsHnd, CORINFO_CLASS_HANDLE *clsRet ) { - printf("getChildType\r\n"); + printf("getChildType not implemented\r\n"); return CORINFO_TYPE_UNDEF; } @@ -1348,7 +1348,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { DWORD size ) { printf("getArrayInitializationData\r\n"); - return NULL; + return nullptr; } // Check Visibility rules. @@ -1375,7 +1375,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_FIELD_HANDLE ftn, /* IN */ const char **moduleName /* OUT */ ) { - printf("getFieldName\r\n"); + printf("getFieldName not implemented\r\n"); return NULL; } @@ -1518,7 +1518,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual void * allocateArray( ULONG cBytes ) { - printf("\r\n"); + printf("allocateArray not implemented\r\n"); return nullptr; } @@ -1571,8 +1571,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_SIG_INFO* sig, /* IN */ CORINFO_ARG_LIST_HANDLE args /* IN */ ) { - //printf("getArgClass\r\n"); - return NULL; + // TODO: Work out correct return type + return sig->retTypeClass; } /***************************************************************************** @@ -1642,7 +1642,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual void getEEInfo( CORINFO_EE_INFO *pEEInfoOut ) { - printf("getEEInfo\r\n"); memset(pEEInfoOut, 0, sizeof(CORINFO_EE_INFO)); pEEInfoOut->inlinedCallFrameInfo.size = 4; @@ -1650,9 +1649,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Returns name of the JIT timer log virtual LPCWSTR getJitTimeLogFilename() { - printf("getJitTimeLogFilename\r\n"); - return NULL; - + return reinterpret_cast("pyjion.log"); } #ifdef RYUJIT_CTPBUILD @@ -1685,9 +1682,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE ftn, /* IN */ const char **moduleName /* OUT */ ) { - *moduleName = "foo"; - //printf("getMethodName %p\r\n", ftn); - return "bar"; + *moduleName = "modulename"; + return "methodname"; } // this function is for debugging only. It returns a value that @@ -1711,44 +1707,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } -#if !defined(RYUJIT_CTPBUILD) - /*************************************************************************/ - // - // Configuration values - Allows querying of the CLR configuration. - // - /*************************************************************************/ - - // Return an integer ConfigValue if any. - // - virtual int getIntConfigValue( - const wchar_t *name, - int defaultValue - ) { - printf("getIntConfigValue\r\n"); - return 0; - } - - // Return a string ConfigValue if any. - // - virtual wchar_t *getStringConfigValue( - const wchar_t *name - ) { - printf("getStringConfigValue\r\n"); - return NULL; - } - - // Free a string ConfigValue returned by the runtime. - // JITs using the getStringConfigValue query are required - // to return the string values to the runtime for deletion. - // this avoid leaking the memory in the JIT. - virtual void freeStringConfigValue( - wchar_t *value - ) { - printf("freeStringConfigValue\r\n"); - } -#endif // RYUJIT_CTPBUILD - - void getAddressOfPInvokeTarget(CORINFO_METHOD_HANDLE method, CORINFO_CONST_LOOKUP * pLookup) { } @@ -1756,6 +1714,9 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { DWORD getJitFlags(CORJIT_FLAGS * flags, DWORD sizeInBytes) { flags->Add(flags->CORJIT_FLAG_SKIP_VERIFICATION); + flags->Add(flags->CORJIT_FLAG_DEBUG_CODE); + flags->Add(flags->CORJIT_FLAG_NO_INLINING); + flags->Add(flags->CORJIT_FLAG_MIN_OPT); return sizeof(CORJIT_FLAGS); } @@ -1767,14 +1728,17 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE resolveVirtualMethod(CORINFO_METHOD_HANDLE virtualMethod, CORINFO_CLASS_HANDLE implementingClass, CORINFO_CONTEXT_HANDLE ownerType) override { + printf("resolveVirtualMethod not defined\r\n"); return nullptr; } CORINFO_METHOD_HANDLE getUnboxedEntry(CORINFO_METHOD_HANDLE ftn, bool *requiresInstMethodTableArg) override { + printf("getUnboxedEntry not defined\r\n"); return nullptr; } CORINFO_CLASS_HANDLE getDefaultEqualityComparerClass(CORINFO_CLASS_HANDLE elemType) override { + printf("getDefaultEqualityComparerClass not defined\r\n"); return nullptr; } @@ -1788,6 +1752,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } PatchpointInfo *getOSRInfo(unsigned int *ilOffset) override { + printf("getOSRInfo not defined\r\n"); return nullptr; } @@ -1796,14 +1761,17 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } LPCWSTR getStringLiteral(CORINFO_MODULE_HANDLE module, unsigned int metaTOK, int *length) override { + printf("getStringLiteral not defined\r\n"); return nullptr; } const char *getClassNameFromMetadata(CORINFO_CLASS_HANDLE cls, const char **namespaceName) override { + printf("getClassNameFromMetadata not defined\r\n"); return nullptr; } CORINFO_CLASS_HANDLE getTypeInstantiationArgument(CORINFO_CLASS_HANDLE cls, unsigned int index) override { + printf("getTypeInstantiationArgument not defined\r\n"); return nullptr; } @@ -1856,6 +1824,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } void *allocateArray(size_t cBytes) override { + printf("allocateArray not defined\r\n"); return nullptr; } @@ -1869,6 +1838,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { const char *getMethodNameFromMetadata(CORINFO_METHOD_HANDLE ftn, const char **className, const char **namespaceName, const char **enclosingClassName) override { + printf("getMethodNameFromMetadata not defined\r\n"); return nullptr; } @@ -1878,6 +1848,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } void *getHelperFtn(CorInfoHelpFunc ftnNum, void **ppIndirection) override { + printf("getHelperFtn not defined\r\n"); return nullptr; } @@ -1886,6 +1857,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } CORINFO_CLASS_HANDLE getStaticFieldCurrentClass(CORINFO_FIELD_HANDLE field, bool *pIsSpeculative) override { + printf("getStaticFieldCurrentClass not defined\r\n"); return nullptr; } @@ -1904,6 +1876,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_METHOD_HANDLE GetDelegateCtor(CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs *pCtorData) override { + printf("GetDelegateCtor not defined\r\n"); return nullptr; } @@ -1934,6 +1907,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } void *allocGCInfo(size_t size) override { + printf("allocGCInfo not defined\r\n"); return nullptr; } From e5cbab68bc93993dbe224f07ee19bb05fd097182 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 13:14:10 +1000 Subject: [PATCH 039/273] Implement LOAD_METHOD --- Pyjion/absint.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index a1de5df1c..feed1553c 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -796,6 +796,9 @@ bool AbstractInterpreter::interpret() { } lastState.push(&Dict); return false; + case LOAD_METHOD: + lastState.push(&Any); + break; default: printf("Unknown unsupported opcode: %s", opcode_name(opcode)); PyErr_Format(PyExc_ValueError, From 4fad593a7bfb262d05d9352a9f2393147e21ef21 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 13:24:02 +1000 Subject: [PATCH 040/273] Implement CALL_METHOD and IS_OP --- Pyjion/absint.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index feed1553c..35ac91271 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -799,8 +799,32 @@ bool AbstractInterpreter::interpret() { case LOAD_METHOD: lastState.push(&Any); break; + case CALL_METHOD: + { + /* LOAD_METHOD could push a NULL value, but (as above) + it doesn't, so instead it just considers the same scenario. + + This is a method call. Stack layout: + + ... | method | self | arg1 | ... | argN + ^- TOP() + ^- (-oparg) + ^- (-oparg-1) + ^- (-oparg-2) + + `self` and `method` will be POPed by call_function. + We'll be passing `oparg + 1` to call_function, to + make it accept the `self` as a first argument. + + TODO : Add call_function opcode? + */ + lastState.push(&Any); // push result. + break; + } + case IS_OP: + lastState.pop(); //right + break; default: - printf("Unknown unsupported opcode: %s", opcode_name(opcode)); PyErr_Format(PyExc_ValueError, "Unknown unsupported opcode: %s", opcode_name(opcode)); return false; From 30df372cf4da2da55a2291bae2d43dab641eaf4a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 13:39:41 +1000 Subject: [PATCH 041/273] Implement ROT_FOUR --- Pyjion/absint.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 35ac91271..cefe3a44d 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -265,6 +265,34 @@ bool AbstractInterpreter::interpret() { lastState.push(second); break; } + case ROT_FOUR: + { + auto top = lastState.pop_no_escape(); + auto second = lastState.pop_no_escape(); + auto third = lastState.pop_no_escape(); + auto fourth = lastState.pop_no_escape(); + + auto sources = AbstractSource::combine( + top.Sources, + AbstractSource::combine(second.Sources, + AbstractSource::combine(third.Sources, fourth.Sources))); + m_opcodeSources[opcodeIndex] = sources; + + if (top.Value->kind() != second.Value->kind() + || top.Value->kind() != third.Value->kind() + || top.Value->kind() != fourth.Value->kind()) { + top.escapes(); + second.escapes(); + third.escapes(); + fourth.escapes(); + } + + lastState.push(top); + lastState.push(fourth); + lastState.push(third); + lastState.push(second); + break; + } case POP_TOP: lastState.pop_no_escape(); break; From f1b83d3678f02a6fca111a9d70c1a3660ad4c1f1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 13:57:31 +1000 Subject: [PATCH 042/273] Update IS_OP --- Pyjion/absint.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index cefe3a44d..81cf16d8c 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -549,7 +549,6 @@ bool AbstractInterpreter::interpret() { lastState.push(&Dict); break; case COMPARE_OP: { - // TODO Implement IS_OP, _CONTAINS_OP auto two = lastState.pop_no_escape(); auto one = lastState.pop_no_escape(); auto binaryRes = one.Value->compare(one.Sources, oparg, two); @@ -814,9 +813,6 @@ bool AbstractInterpreter::interpret() { } lastState.push(&Any); break; - case YIELD_VALUE: - printf("Unsupported situation.."); - return false; case BUILD_CONST_KEY_MAP: lastState.pop(); //keys for (auto i = 0; i < oparg; i++) { @@ -849,9 +845,11 @@ bool AbstractInterpreter::interpret() { lastState.push(&Any); // push result. break; } - case IS_OP: - lastState.pop(); //right - break; + case IS_OP: { + lastState.pop(); + lastState.pop(); + lastState.push(&Bool); + } default: PyErr_Format(PyExc_ValueError, "Unknown unsupported opcode: %s", opcode_name(opcode)); From a41d20b451647fd093bb7e0810b96567a669642e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 13:58:25 +1000 Subject: [PATCH 043/273] Update IS_OP --- Pyjion/absint.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 81cf16d8c..436e4e94b 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -845,11 +845,11 @@ bool AbstractInterpreter::interpret() { lastState.push(&Any); // push result. break; } - case IS_OP: { + case IS_OP: lastState.pop(); lastState.pop(); lastState.push(&Bool); - } + break; default: PyErr_Format(PyExc_ValueError, "Unknown unsupported opcode: %s", opcode_name(opcode)); From e200b9c11d77d991774b5ac2a2107301f3377fec Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 14:22:05 +1000 Subject: [PATCH 044/273] Allow JIT compile of REPL expressions --- Pyjion/pyjit.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index c702116e2..ee6740476 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -143,10 +143,6 @@ static void JitInit() { unordered_map g_pyjionJittedCode; __declspec(dllexport) bool jit_compile(PyCodeObject* code) { - if (strcmp(PyUnicode_AsUTF8(code->co_name), "") == 0) { - return false; - } - auto jittedCode = (PyjionJittedCode *)code->co_extra; #ifdef DEBUG_TRACE @@ -433,8 +429,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { } static bool jit_compile(PyCodeObject* code) { - if (strcmp(PyUnicode_AsUTF8(code->co_name), "") == 0 || - strcmp(PyUnicode_AsUTF8(code->co_name), "") == 0) { + if (strcmp(PyUnicode_AsUTF8(code->co_name), "") == 0) { return false; } #ifdef DEBUG_TRACE From a2a72ab92ab590c8c3e5f9f55d8423df3f54e3f0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 30 Sep 2020 17:50:59 +1000 Subject: [PATCH 045/273] Add more tracing and debug tools. Fix the JitConfig map in the compiler class, fix some POP code --- CMakeLists.txt | 23 +++++-- Pyjion/absint.cpp | 156 ++++++++++++++++++++++------------------------ Pyjion/cee.h | 24 ++++--- Pyjion/ilgen.h | 2 +- Pyjion/jitinfo.h | 3 +- Pyjion/pycomp.cpp | 2 +- 6 files changed, 107 insertions(+), 103 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d9290b44..0db4ddb09 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,16 +16,17 @@ add_compile_options(-fexceptions) add_definitions(-DUSE_STL) add_compile_options(-fvisibility=hidden) -IF(CMAKE_BUILD_TYPE MATCHES DEBUG) - message("Enabling very verbose messages") +IF(CMAKE_BUILD_TYPE MATCHES Debug) + message(STATUS "Enabling very verbose messages") add_definitions(-DDEBUG) -ENDIF(CMAKE_BUILD_TYPE MATCHES DEBUG) +ENDIF(CMAKE_BUILD_TYPE MATCHES Debug) if(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) add_compile_options(-fdeclspec) add_definitions(-DTARGET_UNIX) + message(STATUS "Enabling UNIX Patches") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-null-arithmetic) else(CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -33,6 +34,9 @@ if(NOT WIN32) endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif(NOT WIN32) + +add_definitions(-DPROFILING_SUPPORTED -DUNICODE) + if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") set(IS_64 1) elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64") @@ -46,11 +50,18 @@ endif() if (IS_64) add_definitions(-D_TARGET_AMD64_) add_definitions(-DTARGET_AMD64) + add_definitions(-DTARGET_64BIT) + add_definitions(-DHOST_64BIT) + add_definitions(-DHOST_AMD64) + message(STATUS "Enabling AMD64") if (NOT WIN32) - add_definitions(-DUNIX_AMD64_ABI) # assume 64-bit + add_definitions(-DUNIX_AMD64_ABI) + add_definitions(-DUNIX_AMD64_ABI_ITF) + message(STATUS "Enabling AMD64 ABI") endif() else() add_definitions(-DTARGET_X86) + message(STATUS "Enabling x86") endif() include_directories(CoreCLR/src/coreclr/src/inc CoreCLR/src/coreclr/src/jit) @@ -71,6 +82,10 @@ if (APPLE) set(CLR_CMAKE_HOST_UNIX 1) set(CLR_JIT_LIB "libclrjit.dylib") add_definitions(-D_XOPEN_SOURCE) + add_definitions(-DTARGET_OSX) + + add_definitions(-DFEATURE_SIMD) + add_definitions(-DFEATURE_HW_INTRINSICS) endif() include_directories(CoreCLR/artifacts/bin/coreclr/${CLR_OS_BUILD}/inc) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index 436e4e94b..fcf28b496 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -90,6 +90,15 @@ bool AbstractInterpreter::preprocess() { } switch (byte) { + case POP_BLOCK: + { + if (!blockStarts.empty()) { + auto blockStart = blockStarts.back(); + blockStarts.pop_back(); + m_blockStarts[opcodeIndex] = blockStart.BlockStart; + } + break; + } case EXTENDED_ARG: { curByte += sizeof(_Py_CODEUNIT); @@ -213,20 +222,19 @@ bool AbstractInterpreter::interpret() { auto opcode = GET_OPCODE(curByte); oparg = GET_OPARG(curByte); - + printf("Processing OPCODE %s\n", opcode_name(opcode)); processOpCode: switch (opcode) { - case EXTENDED_ARG: - { + case EXTENDED_ARG: { curByte += sizeof(_Py_CODEUNIT); oparg = (oparg << 8) | GET_OPARG(curByte); opcode = GET_OPCODE(curByte); update_start_state(lastState, curByte); goto processOpCode; } - case NOP: break; - case ROT_TWO: - { + case NOP: + break; + case ROT_TWO: { auto top = lastState.pop_no_escape(); auto second = lastState.pop_no_escape(); @@ -242,15 +250,14 @@ bool AbstractInterpreter::interpret() { lastState.push(second); break; } - case ROT_THREE: - { + case ROT_THREE: { auto top = lastState.pop_no_escape(); auto second = lastState.pop_no_escape(); auto third = lastState.pop_no_escape(); auto sources = AbstractSource::combine( - top.Sources, - AbstractSource::combine(second.Sources, third.Sources)); + top.Sources, + AbstractSource::combine(second.Sources, third.Sources)); m_opcodeSources[opcodeIndex] = sources; if (top.Value->kind() != second.Value->kind() @@ -265,8 +272,7 @@ bool AbstractInterpreter::interpret() { lastState.push(second); break; } - case ROT_FOUR: - { + case ROT_FOUR: { auto top = lastState.pop_no_escape(); auto second = lastState.pop_no_escape(); auto third = lastState.pop_no_escape(); @@ -275,7 +281,7 @@ bool AbstractInterpreter::interpret() { auto sources = AbstractSource::combine( top.Sources, AbstractSource::combine(second.Sources, - AbstractSource::combine(third.Sources, fourth.Sources))); + AbstractSource::combine(third.Sources, fourth.Sources))); m_opcodeSources[opcodeIndex] = sources; if (top.Value->kind() != second.Value->kind() @@ -299,16 +305,14 @@ bool AbstractInterpreter::interpret() { case DUP_TOP: lastState.push(lastState[lastState.stack_size() - 1]); break; - case DUP_TOP_TWO: - { + case DUP_TOP_TWO: { auto top = lastState[lastState.stack_size() - 1]; auto second = lastState[lastState.stack_size() - 2]; lastState.push(second); lastState.push(top); break; } - case LOAD_CONST: - { + case LOAD_CONST: { auto constSource = add_const_source(opcodeIndex, oparg); auto value = AbstractValueWithSources( to_abstract(PyTuple_GetItem(m_code->co_consts, oparg)), @@ -317,18 +321,16 @@ bool AbstractInterpreter::interpret() { lastState.push(value); break; } - case LOAD_FAST: - { + case LOAD_FAST: { auto localSource = add_local_source(opcodeIndex, oparg); auto local = lastState.get_local(oparg); - + local.ValueInfo.Sources = AbstractSource::combine(localSource, local.ValueInfo.Sources); lastState.push(local.ValueInfo); break; } - case STORE_FAST: - { + case STORE_FAST: { auto valueInfo = lastState.pop_no_escape(); m_opcodeSources[opcodeIndex] = valueInfo.Sources; // STORE_FAST doesn't necessarily give us an assigned value because we @@ -339,7 +341,7 @@ bool AbstractInterpreter::interpret() { lastState.replace_local(oparg, AbstractLocalInfo(valueInfo, valueInfo.Value == &Undefined)); } - break; + break; case DELETE_FAST: // We need to box any previous stores so we can delete them... Otherwise // we won't know if we should raise an unbound local error @@ -372,8 +374,7 @@ bool AbstractInterpreter::interpret() { case INPLACE_RSHIFT: case INPLACE_AND: case INPLACE_XOR: - case INPLACE_OR: - { + case INPLACE_OR: { auto two = lastState.pop_no_escape(); auto one = lastState.pop_no_escape(); auto binaryRes = one.Value->binary(one.Sources, opcode, two); @@ -381,15 +382,14 @@ bool AbstractInterpreter::interpret() { // create an intermediate source which will propagate changes up... auto sources = add_intermediate_source(opcodeIndex); AbstractSource::combine( - AbstractSource::combine(one.Sources, two.Sources), - sources - ); + AbstractSource::combine(one.Sources, two.Sources), + sources + ); auto value = AbstractValueWithSources(binaryRes, sources); lastState.push(value); } - break; - case POP_JUMP_IF_FALSE: - { + break; + case POP_JUMP_IF_FALSE: { auto value = lastState.pop_no_escape(); // merge our current state into the branched to location... @@ -406,8 +406,7 @@ bool AbstractInterpreter::interpret() { // we'll continue processing after the jump with our new state... break; } - case POP_JUMP_IF_TRUE: - { + case POP_JUMP_IF_TRUE: { auto value = lastState.pop_no_escape(); // merge our current state into the branched to location... @@ -424,8 +423,7 @@ bool AbstractInterpreter::interpret() { // we'll continue processing after the jump with our new state... break; } - case JUMP_IF_TRUE_OR_POP: - { + case JUMP_IF_TRUE_OR_POP: { auto curState = lastState; if (update_start_state(lastState, oparg)) { queue.push_back(oparg); @@ -437,9 +435,8 @@ bool AbstractInterpreter::interpret() { goto next; } } - break; - case JUMP_IF_FALSE_OR_POP: - { + break; + case JUMP_IF_FALSE_OR_POP: { auto curState = lastState; if (update_start_state(lastState, oparg)) { queue.push_back(oparg); @@ -451,7 +448,7 @@ bool AbstractInterpreter::interpret() { goto next; } } - break; + break; case JUMP_ABSOLUTE: if (update_start_state(lastState, oparg)) { queue.push_back(oparg); @@ -460,14 +457,13 @@ bool AbstractInterpreter::interpret() { // to the following opcodes before we'll process them. goto next; case JUMP_FORWARD: - if (update_start_state(lastState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { - queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); + if (update_start_state(lastState, (size_t) oparg + curByte + sizeof(_Py_CODEUNIT))) { + queue.push_back((size_t) oparg + curByte + sizeof(_Py_CODEUNIT)); } // Done processing this basic block, we'll need to see a branch // to the following opcodes before we'll process them. goto next; - case RETURN_VALUE: - { + case RETURN_VALUE: { // We don't treat returning as escaping as it would just result in a single // boxing over the lifetime of the function. auto retValue = lastState.pop_no_escape(); @@ -482,7 +478,7 @@ bool AbstractInterpreter::interpret() { m_returnValue = m_returnValue->merge_with(retValue.Value); } - goto next; + goto next; case LOAD_NAME: // Used to load __name__ for a class def lastState.push(&Any); @@ -522,8 +518,7 @@ bool AbstractInterpreter::interpret() { } lastState.push(&List); break; - case BUILD_TUPLE: - { + case BUILD_TUPLE: { vector sources; for (int i = 0; i < oparg; i++) { lastState.pop(); @@ -549,17 +544,17 @@ bool AbstractInterpreter::interpret() { lastState.push(&Dict); break; case COMPARE_OP: { - auto two = lastState.pop_no_escape(); - auto one = lastState.pop_no_escape(); - auto binaryRes = one.Value->compare(one.Sources, oparg, two); - auto sources = add_intermediate_source(opcodeIndex); - AbstractSource::combine( - AbstractSource::combine(one.Sources, two.Sources), - sources - ); - auto value = AbstractValueWithSources(binaryRes, sources); - lastState.push(value); - } + auto two = lastState.pop_no_escape(); + auto one = lastState.pop_no_escape(); + auto binaryRes = one.Value->compare(one.Sources, oparg, two); + auto sources = add_intermediate_source(opcodeIndex); + AbstractSource::combine( + AbstractSource::combine(one.Sources, two.Sources), + sources + ); + auto value = AbstractValueWithSources(binaryRes, sources); + lastState.push(value); + } break; case IMPORT_NAME: lastState.pop(); @@ -573,8 +568,7 @@ bool AbstractInterpreter::interpret() { case LOAD_CLOSURE: lastState.push(&Any); break; - case CALL_FUNCTION: - { + case CALL_FUNCTION: { // TODO: Known functions could return known return types. int argCnt = oparg & 0xff; int kwArgCnt = (oparg >> 8) & 0xff; @@ -597,8 +591,7 @@ bool AbstractInterpreter::interpret() { lastState.push(&Any); break; } - case CALL_FUNCTION_KW: - { + case CALL_FUNCTION_KW: { int na = oparg; // Pop the names tuple @@ -628,14 +621,13 @@ bool AbstractInterpreter::interpret() { lastState.push(&Any); break; - case MAKE_FUNCTION: - { + case MAKE_FUNCTION: { lastState.pop(); // qual name lastState.pop(); // code if (oparg & 0x08) { // closure object - lastState.pop(); + lastState.pop(); } if (oparg & 0x04) { // annotations @@ -662,17 +654,16 @@ bool AbstractInterpreter::interpret() { case UNARY_POSITIVE: case UNARY_NEGATIVE: case UNARY_INVERT: - case UNARY_NOT: - { + case UNARY_NOT: { auto one = lastState.pop_no_escape(); auto unaryRes = one.Value->unary(one.Sources, opcode); auto sources = add_intermediate_source(opcodeIndex); AbstractSource::combine( - one.Sources, - sources - ); + one.Sources, + sources + ); // TODO: The code generator currently assumes it is *always* dealing // with an object (i.e. not an unboxed value). Support should be @@ -740,13 +731,12 @@ bool AbstractInterpreter::interpret() { lastState.pop(); lastState.push(&Any); break; - case FOR_ITER: - { + case FOR_ITER: { // For branches out with the value consumed auto leaveState = lastState; leaveState.pop(); - if (update_start_state(leaveState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { - queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); + if (update_start_state(leaveState, (size_t) oparg + curByte + sizeof(_Py_CODEUNIT))) { + queue.push_back((size_t) oparg + curByte + sizeof(_Py_CODEUNIT)); } // When we compile this we don't actually leave the value on the stack, @@ -759,6 +749,7 @@ bool AbstractInterpreter::interpret() { case POP_BLOCK: // Restore the stack state to what we had on entry lastState.m_stack = m_startStates[m_blockStarts[opcodeIndex]].m_stack; + // TODO : Find out what this did, its commented out but it looks important?! //merge_states(m_startStates[m_blockStarts[opcodeIndex]], lastState); break; case POP_EXCEPT: @@ -781,17 +772,16 @@ bool AbstractInterpreter::interpret() { lastState.pop(); lastState.pop(); break; - case SETUP_FINALLY: - { + case SETUP_FINALLY: { auto finallyState = lastState; // Finally is entered with value pushed onto stack indicating reason for // the finally running... finallyState.push(&Any); - if (update_start_state(finallyState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { - queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); + if (update_start_state(finallyState, (size_t) oparg + curByte + sizeof(_Py_CODEUNIT))) { + queue.push_back((size_t) oparg + curByte + sizeof(_Py_CODEUNIT)); } } - break; + break; case FORMAT_VALUE: if ((oparg & FVS_MASK) == FVS_HAVE_SPEC) { // format spec @@ -806,13 +796,15 @@ bool AbstractInterpreter::interpret() { } lastState.push(&String); break; - case SETUP_WITH: - lastState.push(&Any); - if (update_start_state(lastState, (size_t)oparg + curByte + sizeof(_Py_CODEUNIT))) { - queue.push_back((size_t)oparg + curByte + sizeof(_Py_CODEUNIT)); + case SETUP_WITH: { + auto finallyState = lastState; + finallyState.push(&Any); + if (update_start_state(finallyState, (size_t) oparg + curByte + sizeof(_Py_CODEUNIT))) { + queue.push_back((size_t) oparg + curByte + sizeof(_Py_CODEUNIT)); } lastState.push(&Any); break; + } case BUILD_CONST_KEY_MAP: lastState.pop(); //keys for (auto i = 0; i < oparg; i++) { diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 5639f5d62..b27be06d3 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include "openum.h" @@ -54,20 +56,16 @@ using namespace std; class CCorJitHost : public ICorJitHost { protected: - map intSettings; - map strSettings; + map intSettings; + map strSettings; public: CCorJitHost(){ - intSettings = map(); - strSettings = map(); - -#if DEBUG - intSettings = { - "JitLsraStats": 1, - "DumpJittedMethods": 1, - "JitDumpToDebugger": 1 - }; -#endif + // DEBUG settings. + intSettings[u"JitLsraStats"] = 1; + intSettings[u"DumpJittedMethods"] = 1; + intSettings[u"JitDumpToDebugger"] = 1; + intSettings[u"JitDumpASCII"] = 1; + strSettings[u"JitDump"] = u"methodname"; } void * allocateMemory(size_t size) override @@ -91,7 +89,7 @@ public: CCorJitHost(){ { if (strSettings.find(name) != strSettings.end()) return strSettings[name]; - return NULL; + return nullptr; } void freeStringConfigValue(const WCHAR* value) override diff --git a/Pyjion/ilgen.h b/Pyjion/ilgen.h index 2015a50de..58703d6ab 100644 --- a/Pyjion/ilgen.h +++ b/Pyjion/ilgen.h @@ -133,7 +133,7 @@ class ILGenerator { void mark_label(Label label) { auto info = &m_labels[label.m_index]; - _ASSERTE(info->m_location == -1); + assert(info->m_location == -1); info->m_location = (int)m_il.size(); for (int i = 0; i < info->m_branchOffsets.size(); i++) { auto from = info->m_branchOffsets[i]; diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index fb0592c38..d06a7b854 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -1216,8 +1216,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { virtual const char* getHelperName( CorInfoHelpFunc ) { - printf("getHelperName not implemented\r\n"); - return nullptr; + return "AnyJITHelper"; } // This function tries to initialize the class (run the class constructor). diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 8ac42a054..5d2ec4e98 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -24,7 +24,7 @@ */ -#include +#include #include #include "pycomp.h" From ba70d1995dcfb6c3bc02cdc89e03fb1cbb701829 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 10:56:23 +1000 Subject: [PATCH 046/273] Annotate some of the compiler warnings --- Pyjion/absvalue.h | 1 + Pyjion/ipycomp.h | 4 +--- Pyjion/jitinfo.h | 54 +++++++++++++++++++++++------------------------ Pyjion/pycomp.cpp | 1 + Pyjion/pyjit.cpp | 8 +++---- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/Pyjion/absvalue.h b/Pyjion/absvalue.h index da8e022ca..34529389f 100644 --- a/Pyjion/absvalue.h +++ b/Pyjion/absvalue.h @@ -56,6 +56,7 @@ enum AbstractValueKind { }; static bool is_known_type(AbstractValueKind kind) { + // TODO : AVK_Any and AVK_Undefined are missing from here, dont know if this was intentional? switch (kind) { case AVK_Integer: case AVK_Float: diff --git a/Pyjion/ipycomp.h b/Pyjion/ipycomp.h index a4f615744..54acf72be 100644 --- a/Pyjion/ipycomp.h +++ b/Pyjion/ipycomp.h @@ -66,10 +66,8 @@ enum BranchType { class JittedCode { public: - virtual ~JittedCode() { - } + virtual ~JittedCode() = default; virtual void* get_code_addr() = 0; - }; // Defines the interface between the abstract compiler and code generator diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index d06a7b854..b6c2b2d19 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -73,7 +73,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { delete m_module; } - void* get_code_addr() { + void* get_code_addr() override { return m_codeAddr; } @@ -1581,9 +1581,9 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { *****************************************************************************/ // Returns the HRESULT of the current exception - virtual HRESULT GetErrorHRESULT( + HRESULT GetErrorHRESULT( struct _EXCEPTION_POINTERS *pExceptionPointers - ) { + ) override { printf("GetErrorHRESULT\r\n"); return E_FAIL; @@ -1592,10 +1592,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Fetches the message of the current exception // Returns the size of the message (including terminating null). This can be // greater than bufferLength if the buffer is insufficient. - virtual ULONG GetErrorMessage( + ULONG GetErrorMessage( __inout_ecount(bufferLength) LPWSTR buffer, ULONG bufferLength - ) { + ) override { printf("GetErrorMessage\r\n"); return 0; } @@ -1606,27 +1606,27 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // things like ThreadStoppedException ... // returns EXCEPTION_CONTINUE_EXECUTION if exception is fixed up by the EE - virtual int FilterException( + int FilterException( struct _EXCEPTION_POINTERS *pExceptionPointers - ) { + ) override { printf("FilterException\r\n"); return 0; } // Cleans up internal EE tracking when an exception is caught. - virtual void HandleException( + void HandleException( struct _EXCEPTION_POINTERS *pExceptionPointers - ) { + ) override { printf("HandleException\r\n"); } - virtual void ThrowExceptionForJitResult( - HRESULT result) { + void ThrowExceptionForJitResult( + HRESULT result) override { printf("ThrowExceptionForJitResult\r\n"); } //Throws an exception defined by the given throw helper. - virtual void ThrowExceptionForHelper( - const CORINFO_HELPER_DESC * throwHelper) { + void ThrowExceptionForHelper( + const CORINFO_HELPER_DESC * throwHelper) override { printf("ThrowExceptionForHelper\r\n"); } @@ -1638,16 +1638,15 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { *****************************************************************************/ // Return details about EE internal data structures - virtual void getEEInfo( + void getEEInfo( CORINFO_EE_INFO *pEEInfoOut - ) { + ) override { memset(pEEInfoOut, 0, sizeof(CORINFO_EE_INFO)); pEEInfoOut->inlinedCallFrameInfo.size = 4; - } // Returns name of the JIT timer log - virtual LPCWSTR getJitTimeLogFilename() { + LPCWSTR getJitTimeLogFilename() override { return reinterpret_cast("pyjion.log"); } @@ -1667,9 +1666,9 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // this function is for debugging only. Returns method token. // Returns mdMethodDefNil for dynamic methods. - virtual mdMethodDef getMethodDefFromMethod( + mdMethodDef getMethodDefFromMethod( CORINFO_METHOD_HANDLE hMethod - ) { + ) override { printf("getMethodDefFromMethod\r\n"); return 0; } @@ -1677,10 +1676,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // this function is for debugging only. It returns the method name // and if 'moduleName' is non-null, it sets it to something that will // says which method (a class name, or a module name) - virtual const char* getMethodName( + const char* getMethodName( CORINFO_METHOD_HANDLE ftn, /* IN */ const char **moduleName /* OUT */ - ) { + ) override { *moduleName = "modulename"; return "methodname"; } @@ -1688,29 +1687,30 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // this function is for debugging only. It returns a value that // is will always be the same for a given method. It is used // to implement the 'jitRange' functionality - virtual unsigned getMethodHash( + unsigned getMethodHash( CORINFO_METHOD_HANDLE ftn /* IN */ - ) { + ) override { return 0; } // this function is for debugging only. - virtual size_t findNameOfToken( + size_t findNameOfToken( CORINFO_MODULE_HANDLE module, /* IN */ mdToken metaTOK, /* IN */ __out_ecount(FQNameCapacity) char * szFQName, /* OUT */ size_t FQNameCapacity /* IN */ - ) { + ) override { printf("findNameOfToken\r\n"); return 0; } - void getAddressOfPInvokeTarget(CORINFO_METHOD_HANDLE method, CORINFO_CONST_LOOKUP * pLookup) + void getAddressOfPInvokeTarget(CORINFO_METHOD_HANDLE method, CORINFO_CONST_LOOKUP * pLookup) override { + printf("getAddressOfPInvokeTarget\r\n"); } - DWORD getJitFlags(CORJIT_FLAGS * flags, DWORD sizeInBytes) + DWORD getJitFlags(CORJIT_FLAGS * flags, DWORD sizeInBytes) override { flags->Add(flags->CORJIT_FLAG_SKIP_VERIFICATION); flags->Add(flags->CORJIT_FLAG_DEBUG_CODE); diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 5d2ec4e98..7068f019c 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -161,6 +161,7 @@ CorInfoType PythonCompiler::to_clr_type(LocalKind kind) { case LK_Float: return CORINFO_TYPE_DOUBLE; case LK_Int: return CORINFO_TYPE_INT; case LK_Bool: return CORINFO_TYPE_BOOL; + case LK_Pointer: return CORINFO_TYPE_PTR; } return CORINFO_TYPE_NATIVEINT; } diff --git a/Pyjion/pyjit.cpp b/Pyjion/pyjit.cpp index ee6740476..7d0c7d540 100644 --- a/Pyjion/pyjit.cpp +++ b/Pyjion/pyjit.cpp @@ -408,7 +408,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { } #ifdef DEBUG_CALL_TRACE - printf("Invoking default %s from %s line %d %s %p %p\r\n", + printf("Invoking default %s from %s line %d %p %p \r\n", PyUnicode_AsUTF8(frame->f_code->co_name), PyUnicode_AsUTF8(frame->f_code->co_filename), frame->f_code->co_firstlineno, @@ -418,7 +418,7 @@ PyObject* Jit_EvalTrace(PyjionJittedCode* state, PyFrameObject *frame) { #endif auto res = _PyEval_EvalFrameDefault(tstate, frame, 0); #ifdef DEBUG_CALL_TRACE - printf("Returning default %s from %s line %d %s %p\r\n", + printf("Returning default %s from %s line %d %p\r\n", PyUnicode_AsUTF8(frame->f_code->co_name), PyUnicode_AsUTF8(frame->f_code->co_filename), frame->f_code->co_firstlineno, @@ -526,7 +526,7 @@ PyObject* PyJit_EvalFrame(PyThreadState *ts, PyFrameObject *f, int throwflag) { } //SetLastError(err); #ifdef DEBUG_CALL_TRACE - printf("Falling to EFD %s from %s line %d %s %p\r\n", + printf("Falling to EFD %s from %s line %d %p\r\n", PyUnicode_AsUTF8(f->f_code->co_name), PyUnicode_AsUTF8(f->f_code->co_filename), f->f_code->co_firstlineno, @@ -536,7 +536,7 @@ PyObject* PyJit_EvalFrame(PyThreadState *ts, PyFrameObject *f, int throwflag) { auto res = _PyEval_EvalFrameDefault(ts, f, throwflag); #ifdef DEBUG_CALL_TRACE - printf("Returning EFD %s from %s line %d %s %p\r\n", + printf("Returning EFD %s from %s line %d %p\r\n", PyUnicode_AsUTF8(f->f_code->co_name), PyUnicode_AsUTF8(f->f_code->co_filename), f->f_code->co_firstlineno, From 17a05e79d1572ca7cc3517da65be59d46dc1a7bf Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 11:19:57 +1000 Subject: [PATCH 047/273] Big cleanup of the ICorJitInfo implementation methods to use override and remove virtual implementations of methods that no longer exist in the signature --- Pyjion/jitinfo.h | 642 +++++++++++++++++++---------------------------- 1 file changed, 258 insertions(+), 384 deletions(-) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index b6c2b2d19..fcaba9c70 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -81,7 +81,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void freeMem(PVOID code) { } - virtual void allocMem( + void allocMem( ULONG hotCodeSize, /* IN */ ULONG coldCodeSize, /* IN */ ULONG roDataSize, /* IN */ @@ -90,10 +90,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void ** hotCodeBlock, /* OUT */ void ** coldCodeBlock, /* OUT */ void ** roDataBlock /* OUT */ - ) { + ) override { } - virtual BOOL logMsg(unsigned level, const char* fmt, va_list args) { + BOOL logMsg(unsigned level, const char* fmt, va_list args) override { if (level < 7) { #if JIT_FAIL_LOG vprintf(fmt, args); @@ -102,12 +102,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return TRUE; } - int doAssert(const char* szFile, int iLine, const char* szExpr) { + int doAssert(const char* szFile, int iLine, const char* szExpr) override { printf("Assert: %s %d", szFile, iLine); return 0; } - virtual void reportFatalError(CorJitResult result) { + void reportFatalError(CorJitResult result) override { printf("Fatal error %X\r\n", result); } @@ -116,23 +116,23 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // the signature information and method handle the JIT used to lay out the call site. If // the call site has no signature information (e.g. a helper call) or has no method handle // (e.g. a CALLI P/Invoke), then null should be passed instead. - virtual void recordCallSite( + void recordCallSite( ULONG instrOffset, /* IN */ CORINFO_SIG_INFO * callSig, /* IN */ CORINFO_METHOD_HANDLE methodHandle /* IN */ - ) { + ) override { //printf("recordCallSite\r\n"); } #endif // !defined(RYUJIT_CTPBUILD) - virtual void recordRelocation( + void recordRelocation( void * location, /* IN */ void * target, /* IN */ WORD fRelocType, /* IN */ - WORD slotNum = 0, /* IN */ - INT32 addlDelta = 0 /* IN */ - ) { + WORD slotNum, /* IN */ + INT32 addlDelta /* IN */ + ) override { //printf("recordRelocation\r\n"); switch (fRelocType) { case IMAGE_REL_BASED_DIR64: @@ -143,10 +143,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { { target = (BYTE *)target + addlDelta; - INT32 * fixupLocation = (INT32 *)((BYTE *)location + slotNum); + auto * fixupLocation = (INT32 *)((BYTE *)location + slotNum); BYTE * baseAddr = (BYTE *)fixupLocation + sizeof(INT32); - INT64 delta = (INT64)((BYTE *)target - baseAddr); + auto delta = (INT64)((BYTE *)target - baseAddr); // // Do we need to insert a jump stub to make the source reach the target? @@ -186,7 +186,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } } - virtual WORD getRelocTypeHint(void * target) { + WORD getRelocTypeHint(void * target) override { return -1; } @@ -195,7 +195,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // is cross-compiling (such as the case for crossgen), it will return a // different value than if it was compiling for the host architecture. // - virtual DWORD getExpectedTargetArchitecture() { + DWORD getExpectedTargetArchitecture() override { //printf("getExpectedTargetArchitecture\r\n"); #ifdef _TARGET_AMD64_ return IMAGE_FILE_MACHINE_AMD64; @@ -230,57 +230,57 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Return details about EE internal data structures - virtual DWORD getThreadTLSIndex( - void **ppIndirection = NULL - ) { + DWORD getThreadTLSIndex( + void **ppIndirection + ) override { printf("getThreadTLSIndex not implemented\r\n"); return 0; } - virtual const void * getInlinedCallFrameVptr( - void **ppIndirection = NULL - ) { + const void * getInlinedCallFrameVptr( + void **ppIndirection + ) override { printf("getInlinedCallFrameVptr not implemented\r\n"); - return NULL; + return nullptr; } - virtual LONG * getAddrOfCaptureThreadGlobal( - void **ppIndirection = NULL - ) { + LONG * getAddrOfCaptureThreadGlobal( + void **ppIndirection + ) override { printf("getAddrOfCaptureThreadGlobal not implemented\r\n"); - return NULL; + return nullptr; } virtual SIZE_T* getAddrModuleDomainID(CORINFO_MODULE_HANDLE module) { printf("getAddrModuleDomainID not implemented\r\n"); - return 0; + return nullptr; } // return a callable address of the function (native code). This function // may return a different value (depending on whether the method has // been JITed or not. - virtual void getFunctionEntryPoint( + void getFunctionEntryPoint( CORINFO_METHOD_HANDLE ftn, /* IN */ CORINFO_CONST_LOOKUP * pResult, /* OUT */ - CORINFO_ACCESS_FLAGS accessFlags = CORINFO_ACCESS_ANY) { - BaseMethod* method = (BaseMethod*)ftn; + CORINFO_ACCESS_FLAGS accessFlags) override { + auto* method = (BaseMethod*)ftn; method->getFunctionEntryPoint(pResult); } // return a directly callable address. This can be used similarly to the // value returned by getFunctionEntryPoint() except that it is // guaranteed to be multi callable entrypoint. - virtual void getFunctionFixedEntryPoint( + void getFunctionFixedEntryPoint( CORINFO_METHOD_HANDLE ftn, - CORINFO_CONST_LOOKUP * pResult) { + CORINFO_CONST_LOOKUP * pResult) override { printf("getFunctionFixedEntryPoint not implemented\r\n"); } // get the synchronization handle that is passed to monXstatic function - virtual void* getMethodSync( + void* getMethodSync( CORINFO_METHOD_HANDLE ftn, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("getMethodSync not implemented\r\n"); return nullptr; } @@ -302,39 +302,39 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { #else // get slow lazy string literal helper to use (CORINFO_HELP_STRCNS*). // Returns CORINFO_HELP_UNDEF if lazy string literal helper cannot be used. - virtual CorInfoHelpFunc getLazyStringLiteralHelper( + CorInfoHelpFunc getLazyStringLiteralHelper( CORINFO_MODULE_HANDLE handle - ) { + ) override { printf("getLazyStringLiteralHelper\r\n"); return CORINFO_HELP_UNDEF; } #endif - virtual CORINFO_MODULE_HANDLE embedModuleHandle( + CORINFO_MODULE_HANDLE embedModuleHandle( CORINFO_MODULE_HANDLE handle, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("embedModuleHandle not implemented\r\n"); return nullptr; } - virtual CORINFO_CLASS_HANDLE embedClassHandle( + CORINFO_CLASS_HANDLE embedClassHandle( CORINFO_CLASS_HANDLE handle, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("embedClassHandle not implemented\r\n"); return nullptr; } - virtual CORINFO_METHOD_HANDLE embedMethodHandle( + CORINFO_METHOD_HANDLE embedMethodHandle( CORINFO_METHOD_HANDLE handle, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("embedMethodHandle not implemented\r\n"); - *ppIndirection = NULL; + *ppIndirection = nullptr; return handle; } - virtual CORINFO_FIELD_HANDLE embedFieldHandle( + CORINFO_FIELD_HANDLE embedFieldHandle( CORINFO_FIELD_HANDLE handle, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("embedFieldHandle not implemented\r\n"); return nullptr; } @@ -345,74 +345,42 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // code is shared and the token contains generic parameters) // then indicate how the handle should be looked up at run-time. // - virtual void embedGenericHandle( + void embedGenericHandle( CORINFO_RESOLVED_TOKEN * pResolvedToken, BOOL fEmbedParent, // TRUE - embeds parent type handle of the field/method handle - CORINFO_GENERICHANDLE_RESULT * pResult) { + CORINFO_GENERICHANDLE_RESULT * pResult) override { printf("embedGenericHandle not implemented\r\n"); } - // Return information used to locate the exact enclosing type of the current method. - // Used only to invoke .cctor method from code shared across generic instantiations - // !needsRuntimeLookup statically known (enclosing type of method itself) - // needsRuntimeLookup: - // CORINFO_LOOKUP_THISOBJ use vtable pointer of 'this' param - // CORINFO_LOOKUP_CLASSPARAM use vtable hidden param - // CORINFO_LOOKUP_METHODPARAM use enclosing type of method-desc hidden param - virtual CORINFO_LOOKUP_KIND getLocationOfThisType( - CORINFO_METHOD_HANDLE context - ) { - printf("getLocationOfThisType\r\n"); - return CORINFO_LOOKUP_KIND{ FALSE }; - } - - // return the unmanaged target *if method has already been prelinked.* - virtual void* getPInvokeUnmanagedTarget( - CORINFO_METHOD_HANDLE method, - void **ppIndirection = NULL - ) { - printf("getPInvokeUnmanagedTarget not implemented\r\n"); - return nullptr; - } - - // return address of fixup area for late-bound PInvoke calls. - virtual void* getAddressOfPInvokeFixup( - CORINFO_METHOD_HANDLE method, - void **ppIndirection = NULL - ) { - printf("getAddressOfPInvokeFixup not implemented\r\n"); - return nullptr; - } - // Generate a cookie based on the signature that would needs to be passed // to CORINFO_HELP_PINVOKE_CALLI - virtual LPVOID GetCookieForPInvokeCalliSig( + LPVOID GetCookieForPInvokeCalliSig( CORINFO_SIG_INFO* szMetaSig, - void ** ppIndirection = NULL - ) { + void ** ppIndirection + ) override { printf("GetCookieForPInvokeCalliSig not implemented\r\n"); return nullptr; } // returns true if a VM cookie can be generated for it (might be false due to cross-module // inlining, in which case the inlining should be aborted) - virtual bool canGetCookieForPInvokeCalliSig( + bool canGetCookieForPInvokeCalliSig( CORINFO_SIG_INFO* szMetaSig - ) { + ) override { printf("canGetCookieForPInvokeCalliSig\r\n"); return true; } // Gets a handle that is checked to see if the current method is // included in "JustMyCode" - virtual CORINFO_JUST_MY_CODE_HANDLE getJustMyCodeHandle( + CORINFO_JUST_MY_CODE_HANDLE getJustMyCodeHandle( CORINFO_METHOD_HANDLE method, - CORINFO_JUST_MY_CODE_HANDLE**ppIndirection = NULL - ) { - CORINFO_JUST_MY_CODE_HANDLE result = NULL; + CORINFO_JUST_MY_CODE_HANDLE**ppIndirection + ) override { + CORINFO_JUST_MY_CODE_HANDLE result; if (ppIndirection) - *ppIndirection = NULL; - DWORD * pFlagAddr = NULL; + *ppIndirection = nullptr; + DWORD * pFlagAddr = nullptr; result = (CORINFO_JUST_MY_CODE_HANDLE) pFlagAddr; return result; } @@ -420,16 +388,16 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Gets a method handle that can be used to correlate profiling data. // This is the IP of a native method, or the address of the descriptor struct // for IL. Always guaranteed to be unique per process, and not to move. */ - virtual void GetProfilingHandle( + void GetProfilingHandle( BOOL *pbHookFunction, void **pProfilerHandle, BOOL *pbIndirectedHandles - ) { + ) override { printf("GetProfilingHandle\r\n"); } // Returns instructions on how to make the call. See code:CORINFO_CALL_INFO for possible return values. - virtual void getCallInfo( + void getCallInfo( // Token info CORINFO_RESOLVED_TOKEN * pResolvedToken, @@ -444,7 +412,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { //out params (OUT) CORINFO_CALL_INFO *pResult - ) { + ) override { auto method = (BaseMethod*)pResolvedToken->hMethod; pResult->hMethod = (CORINFO_METHOD_HANDLE)method; @@ -456,86 +424,85 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { pResult->accessAllowed = CORINFO_ACCESS_ALLOWED; } - virtual BOOL canAccessFamily(CORINFO_METHOD_HANDLE hCaller, - CORINFO_CLASS_HANDLE hInstanceType) { + BOOL canAccessFamily(CORINFO_METHOD_HANDLE hCaller, + CORINFO_CLASS_HANDLE hInstanceType) override { printf("canAccessFamily \r\n"); return FALSE; } // Returns TRUE if the Class Domain ID is the RID of the class (currently true for every class // except reflection emitted classes and generics) - virtual BOOL isRIDClassDomainID(CORINFO_CLASS_HANDLE cls) { + BOOL isRIDClassDomainID(CORINFO_CLASS_HANDLE cls) override { printf("isRIDClassDomainID \r\n"); return FALSE; } // returns the class's domain ID for accessing shared statics - virtual unsigned getClassDomainID( + unsigned getClassDomainID( CORINFO_CLASS_HANDLE cls, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("getClassDomainID not implemented\r\n"); return 0; } - // return the data's address (for static fields only) - virtual void* getFieldAddress( + void* getFieldAddress( CORINFO_FIELD_HANDLE field, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("getFieldAddress not implemented\r\n"); return nullptr; } // registers a vararg sig & returns a VM cookie for it (which can contain other stuff) - virtual CORINFO_VARARGS_HANDLE getVarArgsHandle( + CORINFO_VARARGS_HANDLE getVarArgsHandle( CORINFO_SIG_INFO *pSig, - void **ppIndirection = NULL - ) { + void **ppIndirection + ) override { printf("getVarArgsHandle not implemented\r\n"); return nullptr; } // returns true if a VM cookie can be generated for it (might be false due to cross-module // inlining, in which case the inlining should be aborted) - virtual bool canGetVarArgsHandle( + bool canGetVarArgsHandle( CORINFO_SIG_INFO *pSig - ) { + ) override { printf("canGetVarArgsHandle\r\n"); return false; } // Allocate a string literal on the heap and return a handle to it - virtual InfoAccessType constructStringLiteral( + InfoAccessType constructStringLiteral( CORINFO_MODULE_HANDLE module, mdToken metaTok, void **ppValue - ) { + ) override { printf("constructStringLiteral\r\n"); return IAT_VALUE; } - virtual InfoAccessType emptyStringLiteral( + InfoAccessType emptyStringLiteral( void **ppValue - ) { + ) override { printf("emptyStringLiteral\r\n"); return IAT_VALUE; } // return flags (defined above, CORINFO_FLG_PUBLIC ...) - virtual DWORD getMethodAttribs( + DWORD getMethodAttribs( CORINFO_METHOD_HANDLE ftn /* IN */ - ) { + ) override { auto method = (BaseMethod*)ftn; return method->get_method_attrs(); } // sets private JIT flags, which can be, retrieved using getAttrib. - virtual void setMethodAttribs( + void setMethodAttribs( CORINFO_METHOD_HANDLE ftn, /* IN */ CorInfoMethodRuntimeFlags attribs /* IN */ - ) { + ) override { printf("setMethodAttribs not implemented\r\n"); } @@ -543,12 +510,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // // 'memberParent' is typically only set when verifying. It should be the // result of calling getMemberParent. - virtual void getMethodSig( + void getMethodSig( CORINFO_METHOD_HANDLE ftn, /* IN */ CORINFO_SIG_INFO *sig, /* OUT */ - CORINFO_CLASS_HANDLE memberParent = NULL /* IN */ - ) { - BaseMethod* m = (BaseMethod*)ftn; + CORINFO_CLASS_HANDLE memberParent /* IN */ + ) override { + auto* m = (BaseMethod*)ftn; //printf("getMethodSig %p\r\n", ftn); m->findSig(sig); } @@ -562,10 +529,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // return information about a method private to the implementation // returns false if method is not IL, or is otherwise unavailable. // This method is used to fetch data needed to inline functions - virtual bool getMethodInfo( + bool getMethodInfo( CORINFO_METHOD_HANDLE ftn, /* IN */ CORINFO_METHOD_INFO* info /* OUT */ - ) { + ) override { printf("getMethodInfo not implemented\r\n"); return false; } @@ -578,11 +545,11 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // // The inlined method need not be verified - virtual CorInfoInline canInline( + CorInfoInline canInline( CORINFO_METHOD_HANDLE callerHnd, /* IN */ CORINFO_METHOD_HANDLE calleeHnd, /* IN */ DWORD* pRestrictions /* OUT */ - ) { + ) override { printf("canInline\r\n"); return INLINE_PASS; } @@ -590,10 +557,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Reports whether or not a method can be inlined, and why. canInline is responsible for reporting all // inlining results when it returns INLINE_FAIL and INLINE_NEVER. All other results are reported by the // JIT. - virtual void reportInliningDecision(CORINFO_METHOD_HANDLE inlinerHnd, + void reportInliningDecision(CORINFO_METHOD_HANDLE inlinerHnd, CORINFO_METHOD_HANDLE inlineeHnd, CorInfoInline inlineResult, - const char * reason) { + const char * reason) override { //printf("reportInliningDecision\r\n"); } @@ -601,68 +568,56 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Returns false if the call is across security boundaries thus we cannot tailcall // // The callerHnd must be the immediate caller (i.e. when we have a chain of inlined calls) - virtual bool canTailCall( + bool canTailCall( CORINFO_METHOD_HANDLE callerHnd, /* IN */ CORINFO_METHOD_HANDLE declaredCalleeHnd, /* IN */ CORINFO_METHOD_HANDLE exactCalleeHnd, /* IN */ bool fIsTailPrefix /* IN */ - ) { + ) override { return FALSE; } // Reports whether or not a method can be tail called, and why. // canTailCall is responsible for reporting all results when it returns // false. All other results are reported by the JIT. - virtual void reportTailCallDecision(CORINFO_METHOD_HANDLE callerHnd, + void reportTailCallDecision(CORINFO_METHOD_HANDLE callerHnd, CORINFO_METHOD_HANDLE calleeHnd, bool fIsTailPrefix, CorInfoTailCall tailCallResult, - const char * reason) { + const char * reason) override { //printf("reportTailCallDecision\r\n"); } // get individual exception handler - virtual void getEHinfo( + void getEHinfo( CORINFO_METHOD_HANDLE ftn, /* IN */ unsigned EHnumber, /* IN */ CORINFO_EH_CLAUSE* clause /* OUT */ - ) { + ) override { printf("getEHinfo\r\n"); } // return class it belongs to - virtual CORINFO_CLASS_HANDLE getMethodClass( + CORINFO_CLASS_HANDLE getMethodClass( CORINFO_METHOD_HANDLE method - ) { + ) override { auto meth = (BaseMethod*)method; return meth->getClass(); } // return module it belongs to - virtual CORINFO_MODULE_HANDLE getMethodModule( + CORINFO_MODULE_HANDLE getMethodModule( CORINFO_METHOD_HANDLE method - ) { + ) override { printf("getMethodModule not implemented\r\n"); return nullptr; } - // This function returns the offset of the specified method in the - // vtable of it's owning class or interface. - virtual void getMethodVTableOffset( - CORINFO_METHOD_HANDLE method, /* IN */ - unsigned* offsetOfIndirection, /* OUT */ - unsigned* offsetAfterIndirection /* OUT */ - ) { - *offsetOfIndirection = 0x1234; - *offsetAfterIndirection = 0x2468; - printf("getMethodVTableOffset\r\n"); - } - // If a method's attributes have (getMethodAttribs) CORINFO_FLG_INTRINSIC set, // getIntrinsicID() returns the intrinsic ID. - virtual CorInfoIntrinsics getIntrinsicID( + CorInfoIntrinsics getIntrinsicID( CORINFO_METHOD_HANDLE method, - bool * pMustExpand = NULL - ) { + bool * pMustExpand + ) override { printf("getIntrinsicID\r\n"); return CORINFO_INTRINSIC_Object_GetType; } @@ -677,29 +632,29 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { #endif // RYUJIT_CTPBUILD // return the unmanaged calling convention for a PInvoke - virtual CorInfoUnmanagedCallConv getUnmanagedCallConv( + CorInfoUnmanagedCallConv getUnmanagedCallConv( CORINFO_METHOD_HANDLE method - ) { + ) override { printf("getUnmanagedCallConv\r\n"); return CORINFO_UNMANAGED_CALLCONV_C; } // return if any marshaling is required for PInvoke methods. Note that // method == 0 => calli. The call site sig is only needed for the varargs or calli case - virtual BOOL pInvokeMarshalingRequired( + BOOL pInvokeMarshalingRequired( CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* callSiteSig - ) { + ) override { printf("pInvokeMarshalingRequired\r\n"); return TRUE; } // Check constraints on method type arguments (only). // The parent class should be checked separately using satisfiesClassConstraints(parent). - virtual BOOL satisfiesMethodConstraints( + BOOL satisfiesMethodConstraints( CORINFO_CLASS_HANDLE parent, // the exact parent of the method CORINFO_METHOD_HANDLE method - ) { + ) override { //printf("satisfiesMethodConstraints\r\n"); return TRUE; } @@ -707,13 +662,13 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Given a delegate target class, a target method parent class, a target method, // a delegate class, check if the method signature is compatible with the Invoke method of the delegate // (under the typical instantiation of any free type variables in the memberref signatures). - virtual BOOL isCompatibleDelegate( + BOOL isCompatibleDelegate( CORINFO_CLASS_HANDLE objCls, /* type of the delegate target, if any */ CORINFO_CLASS_HANDLE methodParentCls, /* exact parent of the target method, if any */ CORINFO_METHOD_HANDLE method, /* (representative) target method, if any */ CORINFO_CLASS_HANDLE delegateCls, /* exact type of the delegate */ BOOL *pfIsOpenDelegate /* is the delegate open */ - ) { + ) override { printf("isCompatibleDelegate\r\n"); return TRUE; } @@ -740,24 +695,24 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } // load and restore the method - virtual void methodMustBeLoadedBeforeCodeIsRun( + void methodMustBeLoadedBeforeCodeIsRun( CORINFO_METHOD_HANDLE method - ) { + ) override { printf("methodMustBeLoadedBeforeCodeIsRun\r\n"); } - virtual CORINFO_METHOD_HANDLE mapMethodDeclToMethodImpl( + CORINFO_METHOD_HANDLE mapMethodDeclToMethodImpl( CORINFO_METHOD_HANDLE method - ) { + ) override { printf("mapMethodDeclToMethodImpl\r\n"); return nullptr; } // Returns the global cookie for the /GS unsafe buffer checks // The cookie might be a constant value (JIT), or a handle to memory location (Ngen) - virtual void getGSCookie( + void getGSCookie( GSCookie * pCookieVal, // OUT GSCookie ** ppCookieVal // OUT - ) { + ) override { *pCookieVal = 0x1234; // TODO: Should be a secure value *ppCookieVal = nullptr; //printf("getGSCookie\r\n"); @@ -865,8 +820,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { /**********************************************************************************/ // Resolve metadata token into runtime method handles. - virtual void resolveToken(/* IN, OUT */ CORINFO_RESOLVED_TOKEN * pResolvedToken) { - Module* mod = (Module*)pResolvedToken->tokenScope; + void resolveToken(/* IN, OUT */ CORINFO_RESOLVED_TOKEN * pResolvedToken) override { + auto* mod = (Module*)pResolvedToken->tokenScope; BaseMethod* method = mod->ResolveMethod(pResolvedToken->token); pResolvedToken->hMethod = (CORINFO_METHOD_HANDLE)method; pResolvedToken->hClass = (CORINFO_CLASS_HANDLE)1; // this just suppresses a JIT assert @@ -884,14 +839,15 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { #endif // MDIL // Signature information about the call sig - virtual void findSig( + void findSig( CORINFO_MODULE_HANDLE module, /* IN */ unsigned sigTOK, /* IN */ CORINFO_CONTEXT_HANDLE context, /* IN */ CORINFO_SIG_INFO *sig /* OUT */ - ) { + ) override { printf("findSig %d\r\n", sigTOK); auto mod = (Module*)module; + // TODO : ResolveMethod is signed, sigTok is unsigned. auto method = mod->ResolveMethod(sigTOK); method->findSig(sig); } @@ -899,17 +855,17 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // for Varargs, the signature at the call site may differ from // the signature at the definition. Thus we need a way of // fetching the call site information - virtual void findCallSiteSig( + void findCallSiteSig( CORINFO_MODULE_HANDLE module, /* IN */ unsigned methTOK, /* IN */ CORINFO_CONTEXT_HANDLE context, /* IN */ CORINFO_SIG_INFO *sig /* OUT */ - ) { + ) override { printf("findCallSiteSig\r\n"); } - virtual CORINFO_CLASS_HANDLE getTokenTypeAsHandle( - CORINFO_RESOLVED_TOKEN * pResolvedToken /* IN */) { + CORINFO_CLASS_HANDLE getTokenTypeAsHandle( + CORINFO_RESOLVED_TOKEN * pResolvedToken /* IN */) override { printf("getTokenTypeAsHandle not implemented\r\n"); return nullptr; } @@ -924,37 +880,22 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // // Checks if the given metadata token is valid - virtual BOOL isValidToken( + BOOL isValidToken( CORINFO_MODULE_HANDLE module, /* IN */ unsigned metaTOK /* IN */ - ) { + ) override { printf("isValidToken\r\n"); return TRUE; } // Checks if the given metadata token is valid StringRef - virtual BOOL isValidStringRef( + BOOL isValidStringRef( CORINFO_MODULE_HANDLE module, /* IN */ unsigned metaTOK /* IN */ - ) { + ) override { printf("isValidStringRef\r\n"); return TRUE; } - virtual BOOL shouldEnforceCallvirtRestriction( - CORINFO_MODULE_HANDLE scope - ) { - printf("shouldEnforceCallvirtRestriction\r\n"); return FALSE; - } -#ifdef MDIL - virtual unsigned getTypeTokenForFieldOrMethod( - unsigned fieldOrMethodToken - ) { - printf("getTypeTokenForFieldOrMethod\r\n"); return 0; - } - - virtual unsigned getTokenForType(CORINFO_CLASS_HANDLE cls) { printf("getTokenForType\r\n"); return 0; } -#endif - /**********************************************************************************/ // // ICorClassInfo @@ -963,51 +904,43 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // If the value class 'cls' is isomorphic to a primitive type it will // return that type, otherwise it will return CORINFO_TYPE_VALUECLASS - virtual CorInfoType asCorInfoType( + CorInfoType asCorInfoType( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("asCorInfoType\r\n"); return CORINFO_TYPE_UNDEF; } // for completeness - virtual const char* getClassName( + const char* getClassName( CORINFO_CLASS_HANDLE cls - ) { + ) override { return "classname"; } - // Append a (possibly truncated) representation of the type cls to the preallocated buffer ppBuf of length pnBufLen // If fNamespace=TRUE, include the namespace/enclosing classes // If fFullInst=TRUE (regardless of fNamespace and fAssembly), include namespace and assembly for any type parameters // If fAssembly=TRUE, suffix with a comma and the full assembly qualification // return size of representation - virtual int appendClassName( + int appendClassName( __deref_inout_ecount(*pnBufLen) WCHAR** ppBuf, int* pnBufLen, CORINFO_CLASS_HANDLE cls, BOOL fNamespace, BOOL fFullInst, BOOL fAssembly - ) { + ) override { printf("appendClassName\r\n"); return 0; } // Quick check whether the type is a value class. Returns the same value as getClassAttribs(cls) & CORINFO_FLG_VALUECLASS, except faster. - virtual BOOL isValueClass(CORINFO_CLASS_HANDLE cls) { printf("isValueClass\r\n"); return FALSE; } - - // If this method returns true, JIT will do optimization to inline the check for - // GetTypeFromHandle(handle) == obj.GetType() - virtual BOOL canInlineTypeCheckWithObjectVTable(CORINFO_CLASS_HANDLE cls) { - printf("canInlineTypeCheckWithObjectVTable\r\n"); - return FALSE; - } + BOOL isValueClass(CORINFO_CLASS_HANDLE cls) override { printf("isValueClass\r\n"); return FALSE; } // return flags (defined above, CORINFO_FLG_PUBLIC ...) - virtual DWORD getClassAttribs( + DWORD getClassAttribs( CORINFO_CLASS_HANDLE cls - ) { + ) override { // TODO : Load from a base class and establish correct attribs. return CORINFO_FLG_VALUECLASS; } @@ -1018,64 +951,68 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // an optimization: the JIT may assume that return buffer pointers for return types for which this predicate // returns TRUE are always stack allocated, and thus, that stores to the GC-pointer fields of such return // buffers do not require GC write barriers. - virtual BOOL isStructRequiringStackAllocRetBuf(CORINFO_CLASS_HANDLE cls) { + BOOL isStructRequiringStackAllocRetBuf(CORINFO_CLASS_HANDLE cls) override { printf("isStructRequiringStackAllocRetBuf\r\n"); return FALSE; } - virtual CORINFO_MODULE_HANDLE getClassModule( + CORINFO_MODULE_HANDLE getClassModule( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getClassModule not implemented\r\n"); return nullptr; } // Returns the assembly that contains the module "mod". - virtual CORINFO_ASSEMBLY_HANDLE getModuleAssembly( + CORINFO_ASSEMBLY_HANDLE getModuleAssembly( CORINFO_MODULE_HANDLE mod - ) { + ) override { printf("getModuleAssembly not implemented\r\n"); return nullptr; } // Returns the name of the assembly "assem". - virtual const char* getAssemblyName( + const char* getAssemblyName( CORINFO_ASSEMBLY_HANDLE assem - ) { + ) override { printf("getAssemblyName not implemented\r\n"); - return nullptr; + return "assem"; } // Allocate and delete process-lifetime objects. Should only be // referred to from static fields, lest a leak occur. // Note that "LongLifetimeFree" does not execute destructors, if "obj" // is an array of a struct type with a destructor. - virtual void* LongLifetimeMalloc(size_t sz) { printf("LongLifetimeMalloc\r\n"); return nullptr; } - virtual void LongLifetimeFree(void* obj) { + void* LongLifetimeMalloc(size_t sz) override { + printf("LongLifetimeMalloc\r\n"); + return nullptr; + } + + void LongLifetimeFree(void* obj) override { printf("LongLifetimeFree\r\n"); } - virtual size_t getClassModuleIdForStatics( + size_t getClassModuleIdForStatics( CORINFO_CLASS_HANDLE cls, CORINFO_MODULE_HANDLE *pModule, void **ppIndirection - ) { + ) override { printf("getClassModuleIdForStatics not implemented\r\n"); return 0; } // return the number of bytes needed by an instance of the class - virtual unsigned getClassSize( + unsigned getClassSize( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getClassSize not implemented\r\n"); return 0; } - virtual unsigned getClassAlignmentRequirement( + unsigned getClassAlignmentRequirement( CORINFO_CLASS_HANDLE cls, - BOOL fDoubleAlignHint = FALSE - ) { + BOOL fDoubleAlignHint + ) override { printf("getClassAlignmentRequirement\r\n"); return 0; } @@ -1089,97 +1026,81 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // to one of the CorInfoGCType values which is the GC type of // the i-th machine word of an object of type 'cls' // returns the number of GC pointers in the array - virtual unsigned getClassGClayout( + unsigned getClassGClayout( CORINFO_CLASS_HANDLE cls, /* IN */ BYTE *gcPtrs /* OUT */ - ) { + ) override { printf("getClassGClayout\r\n"); return 0; } // returns the number of instance fields in a class - virtual unsigned getClassNumInstanceFields( + unsigned getClassNumInstanceFields( CORINFO_CLASS_HANDLE cls /* IN */ - ) { + ) override { printf("getClassNumInstanceFields\r\n"); return 0; } - virtual CORINFO_FIELD_HANDLE getFieldInClass( + CORINFO_FIELD_HANDLE getFieldInClass( CORINFO_CLASS_HANDLE clsHnd, INT num - ) { + ) override { printf("getFieldInClass\r\n"); - return NULL; + return nullptr; } - virtual BOOL checkMethodModifier( + BOOL checkMethodModifier( CORINFO_METHOD_HANDLE hMethod, LPCSTR modifier, BOOL fOptional - ) { + ) override { printf("checkMethodModifier\r\n"); return FALSE; } - // returns the "NEW" helper optimized for "newCls." - virtual CorInfoHelpFunc getNewHelper( - CORINFO_RESOLVED_TOKEN * pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle - ) { - printf("getNewHelper\r\n"); - return CORINFO_HELP_UNDEF; - } - // returns the newArr (1-Dim array) helper optimized for "arrayCls." - virtual CorInfoHelpFunc getNewArrHelper( + CorInfoHelpFunc getNewArrHelper( CORINFO_CLASS_HANDLE arrayCls - ) { + ) override { printf("getNewArrHelper\r\n"); return CORINFO_HELP_UNDEF; } // returns the optimized "IsInstanceOf" or "ChkCast" helper - virtual CorInfoHelpFunc getCastingHelper( + CorInfoHelpFunc getCastingHelper( CORINFO_RESOLVED_TOKEN * pResolvedToken, bool fThrowing - ) { + ) override { printf("getCastingHelper\r\n"); return CORINFO_HELP_UNDEF; } // returns helper to trigger static constructor - virtual CorInfoHelpFunc getSharedCCtorHelper( + CorInfoHelpFunc getSharedCCtorHelper( CORINFO_CLASS_HANDLE clsHnd - ) { + ) override { printf("getSharedCCtorHelper\r\n"); return CORINFO_HELP_UNDEF; } - virtual CorInfoHelpFunc getSecurityPrologHelper( - CORINFO_METHOD_HANDLE ftn - ) { - printf("getSecurityPrologHelper\r\n"); - return CORINFO_HELP_UNDEF; - } - // This is not pretty. Boxing nullable actually returns // a boxed not a boxed Nullable. This call allows the verifier // to call back to the EE on the 'box' instruction and get the transformed // type to use for verification. - virtual CORINFO_CLASS_HANDLE getTypeForBox( + CORINFO_CLASS_HANDLE getTypeForBox( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getTypeForBox not implemented\r\n"); - return NULL; + return nullptr; } // returns the correct box helper for a particular class. Note // that if this returns CORINFO_HELP_BOX, the JIT can assume // 'standard' boxing (allocate object and copy), and optimize - virtual CorInfoHelpFunc getBoxHelper( + CorInfoHelpFunc getBoxHelper( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getBoxHelper\r\n"); return CORINFO_HELP_UNDEF; } @@ -1196,9 +1117,9 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // The EE set 'helperCopies' on return to indicate what kind of // helper has been created. - virtual CorInfoHelpFunc getUnBoxHelper( + CorInfoHelpFunc getUnBoxHelper( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getUnBoxHelper\r\n"); return CORINFO_HELP_UNDEF; } @@ -1213,28 +1134,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } #endif - virtual const char* getHelperName( + const char* getHelperName( CorInfoHelpFunc - ) { + ) override { return "AnyJITHelper"; } - // This function tries to initialize the class (run the class constructor). - // this function returns whether the JIT must insert helper calls before - // accessing static field or method. - // - // See code:ICorClassInfo#ClassConstruction. - virtual CorInfoInitClassResult initClass( - CORINFO_FIELD_HANDLE field, // Non-NULL - inquire about cctor trigger before static field access - // NULL - inquire about cctor trigger in method prolog - CORINFO_METHOD_HANDLE method, // Method referencing the field or prolog - CORINFO_CONTEXT_HANDLE context, // Exact context of method - BOOL speculative = FALSE // TRUE means don't actually run it - ) { - //printf("initClass\r\n"); - return CORINFO_INITCLASS_NOT_REQUIRED; - } - // This used to be called "loadClass". This records the fact // that the class must be loaded (including restored if necessary) before we execute the // code that we are currently generating. When jitting code @@ -1245,52 +1150,52 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // This is typically used to ensure value types are loaded before zapped // code that manipulates them is executed, so that the GC can access information // about those value types. - virtual void classMustBeLoadedBeforeCodeIsRun( + void classMustBeLoadedBeforeCodeIsRun( CORINFO_CLASS_HANDLE cls - ) { + ) override { //printf("classMustBeLoadedBeforeCodeIsRun\r\n"); } // returns the class handle for the special builtin classes - virtual CORINFO_CLASS_HANDLE getBuiltinClass( + CORINFO_CLASS_HANDLE getBuiltinClass( CorInfoClassId classId - ) { + ) override { printf("getBuiltinClass\r\n"); - return NULL; + return nullptr; } // "System.Int32" ==> CORINFO_TYPE_INT.. - virtual CorInfoType getTypeForPrimitiveValueClass( + CorInfoType getTypeForPrimitiveValueClass( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getTypeForPrimitiveValueClass\r\n"); return CORINFO_TYPE_UNDEF; } // TRUE if child is a subtype of parent // if parent is an interface, then does child implement / extend parent - virtual BOOL canCast( + BOOL canCast( CORINFO_CLASS_HANDLE child, // subtype (extends parent) CORINFO_CLASS_HANDLE parent // base type - ) { + ) override { printf("canCast\r\n"); return TRUE; } // TRUE if cls1 and cls2 are considered equivalent types. - virtual BOOL areTypesEquivalent( + BOOL areTypesEquivalent( CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2 - ) { + ) override { printf("areTypesEquivalent\r\n"); return FALSE; } // returns is the intersection of cls1 and cls2. - virtual CORINFO_CLASS_HANDLE mergeClasses( + CORINFO_CLASS_HANDLE mergeClasses( CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2 - ) { + ) override { printf("mergeClasses not implemented\r\n"); return nullptr; } @@ -1298,9 +1203,9 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Given a class handle, returns the Parent type. // For COMObjectType, it returns Class Handle of System.Object. // Returns 0 if System.Object is passed in. - virtual CORINFO_CLASS_HANDLE getParentType( + CORINFO_CLASS_HANDLE getParentType( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getParentType not implemented\r\n"); return nullptr; } @@ -1309,54 +1214,54 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // not a primitive type, *clsRet will be set. // Given an Array of Type Foo, returns Foo. // Given BYREF Foo, returns Foo - virtual CorInfoType getChildType( + CorInfoType getChildType( CORINFO_CLASS_HANDLE clsHnd, CORINFO_CLASS_HANDLE *clsRet - ) { + ) override { printf("getChildType not implemented\r\n"); return CORINFO_TYPE_UNDEF; } // Check constraints on type arguments of this class and parent classes - virtual BOOL satisfiesClassConstraints( + BOOL satisfiesClassConstraints( CORINFO_CLASS_HANDLE cls - ) { + ) override { //printf("satisfiesClassConstraints\r\n"); return TRUE; } // Check if this is a single dimensional array type - virtual BOOL isSDArray( + BOOL isSDArray( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("isSDArray\r\n"); return TRUE; } // Get the numbmer of dimensions in an array - virtual unsigned getArrayRank( + unsigned getArrayRank( CORINFO_CLASS_HANDLE cls - ) { + ) override { printf("getArrayRank\r\n"); return 0; } // Get static field data for an array - virtual void * getArrayInitializationData( + void * getArrayInitializationData( CORINFO_FIELD_HANDLE field, DWORD size - ) { + ) override { printf("getArrayInitializationData\r\n"); return nullptr; } // Check Visibility rules. - virtual CorInfoIsAccessAllowedResult canAccessClass( + CorInfoIsAccessAllowedResult canAccessClass( CORINFO_RESOLVED_TOKEN * pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_HELPER_DESC *pAccessHelper /* If canAccessMethod returns something other than ALLOWED, then this is filled in. */ - ) { + ) override { printf("canAccessClass\r\n"); return CORINFO_ACCESS_ALLOWED; } @@ -1370,20 +1275,20 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // this function is for debugging only. It returns the field name // and if 'moduleName' is non-null, it sets it to something that will // says which method (a class name, or a module name) - virtual const char* getFieldName( + const char* getFieldName( CORINFO_FIELD_HANDLE ftn, /* IN */ const char **moduleName /* OUT */ - ) { + ) override { printf("getFieldName not implemented\r\n"); - return NULL; + return "field"; } // return class it belongs to - virtual CORINFO_CLASS_HANDLE getFieldClass( + CORINFO_CLASS_HANDLE getFieldClass( CORINFO_FIELD_HANDLE field - ) { + ) override { printf("getFieldClass\r\n"); - return 0; + return nullptr; } // Return the field's type, if it is CORINFO_TYPE_VALUECLASS 'structType' is set @@ -1392,37 +1297,28 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // // 'memberParent' is typically only set when verifying. It should be the // result of calling getMemberParent. - virtual CorInfoType getFieldType( + CorInfoType getFieldType( CORINFO_FIELD_HANDLE field, CORINFO_CLASS_HANDLE *structType, - CORINFO_CLASS_HANDLE memberParent = NULL /* IN */ - ) { + CORINFO_CLASS_HANDLE memberParent /* IN */ + ) override { printf("getFieldType\r\n"); return CORINFO_TYPE_UNDEF; } // return the data member's instance offset - virtual unsigned getFieldOffset( + unsigned getFieldOffset( CORINFO_FIELD_HANDLE field - ) { + ) override { printf("getFieldOffset\r\n"); return 0; } - // TODO: jit64 should be switched to the same plan as the i386 jits - use - // getClassGClayout to figure out the need for writebarrier helper, and inline the copying. - // The interpretted value class copy is slow. Once this happens, USE_WRITE_BARRIER_HELPERS - virtual bool isWriteBarrierHelperRequired( - CORINFO_FIELD_HANDLE field) { - printf("isWriteBarrierHelperRequired\r\n"); - return false; - } - - virtual void getFieldInfo(CORINFO_RESOLVED_TOKEN * pResolvedToken, + void getFieldInfo(CORINFO_RESOLVED_TOKEN * pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_ACCESS_FLAGS flags, CORINFO_FIELD_INFO *pResult - ) { + ) override { printf("\r\n"); } #ifdef MDIL @@ -1433,7 +1329,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { #endif // Returns true iff "fldHnd" represents a static field. - virtual bool isFieldStatic(CORINFO_FIELD_HANDLE fldHnd) { + bool isFieldStatic(CORINFO_FIELD_HANDLE fldHnd) override { printf("isFieldStatic\r\n"); return FALSE; } @@ -1451,13 +1347,13 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Note that unless CORJIT_FLG_DEBUG_CODE is specified, this function will // be used only as a hint and the native compiler should not change its // code generation. - virtual void getBoundaries( + void getBoundaries( CORINFO_METHOD_HANDLE ftn, // [IN] method of interest unsigned int *cILOffsets, // [OUT] size of pILOffsets DWORD **pILOffsets, // [OUT] IL offsets of interest // jit MUST free with freeArray! ICorDebugInfo::BoundaryTypes *implictBoundaries // [OUT] tell jit, all boundries of this type - ) { + ) override { printf("getBoundaries\r\n"); } @@ -1468,12 +1364,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Note that debugger (and profiler) is assuming that all of the // offsets form a contiguous block of memory, and that the // OffsetMapping is sorted in order of increasing native offset. - virtual void setBoundaries( + void setBoundaries( CORINFO_METHOD_HANDLE ftn, // [IN] method of interest ULONG32 cMap, // [IN] size of pMap ICorDebugInfo::OffsetMapping *pMap // [IN] map including all points of interest. // jit allocated with allocateArray, EE frees - ) { + ) override { printf("setBoundaries\r\n"); } @@ -1485,14 +1381,14 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // Note that unless CORJIT_FLG_DEBUG_CODE is specified, this function will // be used only as a hint and the native compiler should not change its // code generation. - virtual void getVars( + void getVars( CORINFO_METHOD_HANDLE ftn, // [IN] method of interest ULONG32 *cVars, // [OUT] size of 'vars' ICorDebugInfo::ILVarInfo **vars, // [OUT] scopes of variables of interest // jit MUST free with freeArray! bool *extendOthers // [OUT] it TRUE, then assume the scope // of unmentioned vars is entire method - ) { + ) override { printf("getVars\r\n"); } @@ -1500,34 +1396,24 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // note that the JIT might split lifetimes into different // locations etc. - virtual void setVars( + void setVars( CORINFO_METHOD_HANDLE ftn, // [IN] method of interest ULONG32 cVars, // [IN] size of 'vars' ICorDebugInfo::NativeVarInfo *vars // [IN] map telling where local vars are stored at what points // jit allocated with allocateArray, EE frees - ) { + ) override { printf("setVars\r\n"); } /*-------------------------- Misc ---------------------------------------*/ - // Used to allocate memory that needs to handed to the EE. - // For eg, use this to allocated memory for reporting debug info, - // which will be handed to the EE by setVars() and setBoundaries() - virtual void * allocateArray( - ULONG cBytes - ) { - printf("allocateArray not implemented\r\n"); - return nullptr; - } - // JitCompiler will free arrays passed by the EE using this // For eg, The EE returns memory in getVars() and getBoundaries() // to the JitCompiler, which the JitCompiler should release using // freeArray() - virtual void freeArray( + void freeArray( void *array - ) { + ) override { printf("freeArray\r\n"); } @@ -1539,37 +1425,18 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { // advance the pointer to the argument list. // a ptr of 0, is special and always means the first argument - virtual CORINFO_ARG_LIST_HANDLE getArgNext( + CORINFO_ARG_LIST_HANDLE getArgNext( CORINFO_ARG_LIST_HANDLE args /* IN */ - ) { - //printf("getArgNext %p\r\n", args); + ) override { + printf("getArgNext %p\r\n", args); return (CORINFO_ARG_LIST_HANDLE)(((Parameter*)args) + 1); } - // Get the type of a particular argument - // CORINFO_TYPE_UNDEF is returned when there are no more arguments - // If the type returned is a primitive type (or an enum) *vcTypeRet set to NULL - // otherwise it is set to the TypeHandle associted with the type - // Enumerations will always look their underlying type (probably should fix this) - // Otherwise vcTypeRet is the type as would be seen by the IL, - // The return value is the type that is used for calling convention purposes - // (Thus if the EE wants a value class to be passed like an int, then it will - // return CORINFO_TYPE_INT - virtual CorInfoTypeWithMod getArgType( - CORINFO_SIG_INFO* sig, /* IN */ - CORINFO_ARG_LIST_HANDLE args, /* IN */ - CORINFO_CLASS_HANDLE *vcTypeRet /* OUT */ - ) { - //printf("getArgType %p\r\n", args); - *vcTypeRet = nullptr; - return (CorInfoTypeWithMod)((Parameter*)args)->m_type; - } - // If the Arg is a CORINFO_TYPE_CLASS fetch the class handle associated with it - virtual CORINFO_CLASS_HANDLE getArgClass( + CORINFO_CLASS_HANDLE getArgClass( CORINFO_SIG_INFO* sig, /* IN */ CORINFO_ARG_LIST_HANDLE args /* IN */ - ) { + ) override { // TODO: Work out correct return type return sig->retTypeClass; } @@ -1586,7 +1453,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { ) override { printf("GetErrorHRESULT\r\n"); return E_FAIL; - } // Fetches the message of the current exception @@ -1721,7 +1587,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void getMethodVTableOffset(CORINFO_METHOD_HANDLE method, unsigned int *offsetOfIndirection, unsigned int *offsetAfterIndirection, bool *isRelative) override { - + // TODO : API added isRelative flag, this doesn't inspect that. + *offsetOfIndirection = 0x1234; + *offsetAfterIndirection = 0x2468; + printf("getMethodVTableOffset\r\n"); } CORINFO_METHOD_HANDLE @@ -1927,6 +1796,11 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return 0; } + CorInfoTypeWithMod + getArgType(CORINFO_SIG_INFO *sig, CORINFO_ARG_LIST_HANDLE args, CORINFO_CLASS_HANDLE *vcTypeRet) override { + return CORINFO_TYPE_MASK; + } + }; #endif From 4d25c8cfe7649436f1293d4b6619a674c31c311c Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 11:50:49 +1000 Subject: [PATCH 048/273] Updated more of the interface bindings --- Pyjion/jitinfo.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index fcaba9c70..39cb33419 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -1287,7 +1287,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_CLASS_HANDLE getFieldClass( CORINFO_FIELD_HANDLE field ) override { - printf("getFieldClass\r\n"); + printf("getFieldClass not implemented\r\n"); return nullptr; } @@ -1438,7 +1438,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_ARG_LIST_HANDLE args /* IN */ ) override { // TODO: Work out correct return type - return sig->retTypeClass; + printf("getArgClass not implemented %p\r\n", args); + return (CORINFO_CLASS_HANDLE)1; } /***************************************************************************** @@ -1798,7 +1799,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CorInfoTypeWithMod getArgType(CORINFO_SIG_INFO *sig, CORINFO_ARG_LIST_HANDLE args, CORINFO_CLASS_HANDLE *vcTypeRet) override { - return CORINFO_TYPE_MASK; + return static_cast(CORINFO_TYPE_REFANY); } }; From a532f555e586a0e2c300e3365e9057e04da4784b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 11:55:40 +1000 Subject: [PATCH 049/273] Fixup some of the interface changes --- Pyjion/jitinfo.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 39cb33419..b7bdd0979 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -1438,8 +1438,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_ARG_LIST_HANDLE args /* IN */ ) override { // TODO: Work out correct return type - printf("getArgClass not implemented %p\r\n", args); - return (CORINFO_CLASS_HANDLE)1; + return nullptr; } /***************************************************************************** @@ -1799,7 +1798,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CorInfoTypeWithMod getArgType(CORINFO_SIG_INFO *sig, CORINFO_ARG_LIST_HANDLE args, CORINFO_CLASS_HANDLE *vcTypeRet) override { - return static_cast(CORINFO_TYPE_REFANY); + *vcTypeRet = nullptr; + return (CorInfoTypeWithMod)((Parameter*)args)->m_type; } }; From 0e11324a6caf891716abe56a3194282635e7795f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 13:21:20 +1000 Subject: [PATCH 050/273] Implement freeMem, allocGCInfo and allocMem --- Pyjion/jitinfo.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index b7bdd0979..37831f8ba 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -79,6 +79,8 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void freeMem(PVOID code) { + // TODO: Validate + PyMem_Free(code); } void allocMem( @@ -91,6 +93,10 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void ** coldCodeBlock, /* OUT */ void ** roDataBlock /* OUT */ ) override { + *hotCodeBlock = PyMem_Malloc(hotCodeSize); + *coldCodeBlock = PyMem_Malloc(coldCodeSize); + *roDataBlock = PyMem_Malloc(roDataSize); + // TODO : Honor flag (alignment.) } BOOL logMsg(unsigned level, const char* fmt, va_list args) override { @@ -1428,7 +1434,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_ARG_LIST_HANDLE getArgNext( CORINFO_ARG_LIST_HANDLE args /* IN */ ) override { - printf("getArgNext %p\r\n", args); return (CORINFO_ARG_LIST_HANDLE)(((Parameter*)args) + 1); } @@ -1775,8 +1780,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } void *allocGCInfo(size_t size) override { - printf("allocGCInfo not defined\r\n"); - return nullptr; + return PyMem_Malloc(size); } void setEHcount(unsigned int cEH) override { From ec4432bb2cb742851327823b1d8d5b1a9cbd84fc Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 14:04:11 +1000 Subject: [PATCH 051/273] Additional output during debug --- Pyjion/jitinfo.h | 79 +++++++++++------------------------------------- 1 file changed, 17 insertions(+), 62 deletions(-) diff --git a/Pyjion/jitinfo.h b/Pyjion/jitinfo.h index 37831f8ba..45ffc701c 100644 --- a/Pyjion/jitinfo.h +++ b/Pyjion/jitinfo.h @@ -109,29 +109,18 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } int doAssert(const char* szFile, int iLine, const char* szExpr) override { - printf("Assert: %s %d", szFile, iLine); + printf(".NET failed assertion: %s %d", szFile, iLine); + // TODO : Use native warnings when it doesn't cause a recursive error. + // PyErr_WarnFormat(PyExc_RuntimeWarning, 1, ".NET failed assertion: %s %d", szFile, iLine); return 0; } void reportFatalError(CorJitResult result) override { - printf("Fatal error %X\r\n", result); + printf("Fatal error from .NET JIT %X\r\n", result); + // TODO : Enable when successful + // PyErr_Format(PyExc_ValueError, "Fatal error from .NET JIT %X\r\n", result); } -#if !defined(RYUJIT_CTPBUILD) - // Associates a native call site, identified by its offset in the native code stream, with - // the signature information and method handle the JIT used to lay out the call site. If - // the call site has no signature information (e.g. a helper call) or has no method handle - // (e.g. a CALLI P/Invoke), then null should be passed instead. - void recordCallSite( - ULONG instrOffset, /* IN */ - CORINFO_SIG_INFO * callSig, /* IN */ - CORINFO_METHOD_HANDLE methodHandle /* IN */ - ) override { - //printf("recordCallSite\r\n"); - } - -#endif // !defined(RYUJIT_CTPBUILD) - void recordRelocation( void * location, /* IN */ void * target, /* IN */ @@ -291,21 +280,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return nullptr; } -#if defined(RYUJIT_CTPBUILD) - // These entry points must be called if a handle is being embedded in - // the code to be passed to a JIT helper function. (as opposed to just - // being passed back into the ICorInfo interface.) - - // a module handle may not always be available. A call to embedModuleHandle should always - // be preceeded by a call to canEmbedModuleHandleForHelper. A dynamicMethod does not have a module - virtual bool canEmbedModuleHandleForHelper( - CORINFO_MODULE_HANDLE handle - ) { - printf("canEmbedModuleHandleForHelper\r\n"); - return FALSE; - } - -#else // get slow lazy string literal helper to use (CORINFO_HELP_STRCNS*). // Returns CORINFO_HELP_UNDEF if lazy string literal helper cannot be used. CorInfoHelpFunc getLazyStringLiteralHelper( @@ -313,7 +287,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { ) override { printf("getLazyStringLiteralHelper\r\n"); return CORINFO_HELP_UNDEF; } -#endif + CORINFO_MODULE_HANDLE embedModuleHandle( CORINFO_MODULE_HANDLE handle, void **ppIndirection @@ -627,16 +601,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { printf("getIntrinsicID\r\n"); return CORINFO_INTRINSIC_Object_GetType; } -#ifndef RYUJIT_CTPBUILD - // Is the given module the System.Numerics.Vectors module? - // This defaults to false. - virtual bool isInSIMDModule( - CORINFO_CLASS_HANDLE classHnd - ) { - return false; - } -#endif // RYUJIT_CTPBUILD - // return the unmanaged calling convention for a PInvoke CorInfoUnmanagedCallConv getUnmanagedCallConv( CORINFO_METHOD_HANDLE method @@ -1130,16 +1094,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return CORINFO_HELP_UNDEF; } -#ifndef RYUJIT_CTPBUILD - virtual void getReadyToRunHelper( - CORINFO_RESOLVED_TOKEN * pResolvedToken, - CorInfoHelpFunc id, - CORINFO_CONST_LOOKUP * pLookup - ) { - printf("getReadyToRunHelper\r\n"); - } -#endif - const char* getHelperName( CorInfoHelpFunc ) override { @@ -1443,6 +1397,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { CORINFO_ARG_LIST_HANDLE args /* IN */ ) override { // TODO: Work out correct return type + printf("getArgClass not implemented\r\n"); return nullptr; } @@ -1521,14 +1476,6 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return reinterpret_cast("pyjion.log"); } -#ifdef RYUJIT_CTPBUILD - // Logs a SQM event for a JITting a very large method. - virtual void logSQMLongJitEvent(unsigned mcycles, unsigned msec, unsigned ilSize, unsigned numBasicBlocks, bool minOpts, - CORINFO_METHOD_HANDLE methodHnd) { - printf("logSQMLongJitEvent\r\n"); - } -#endif // RYUJIT_CTPBUILD - /*********************************************************************************/ // // Diagnostic methods @@ -1776,7 +1723,7 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { void allocUnwindInfo(BYTE *pHotCode, BYTE *pColdCode, ULONG startOffset, ULONG endOffset, ULONG unwindSize, BYTE *pUnwindBlock, CorJitFuncKind funcKind) override { - + printf("allocUnwindInfo not implemented \r\n"); } void *allocGCInfo(size_t size) override { @@ -1792,11 +1739,13 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { } HRESULT allocMethodBlockCounts(UINT32 count, BlockCounts **pBlockCounts) override { + printf("allocMethodBlockCounts not implemented \r\n"); return 0; } HRESULT getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd, UINT32 *pCount, BlockCounts **pBlockCounts, UINT32 *pNumRuns) override { + printf("getMethodBlockCounts not implemented \r\n"); return 0; } @@ -1806,6 +1755,12 @@ class CorJitInfo : public ICorJitInfo, public JittedCode { return (CorInfoTypeWithMod)((Parameter*)args)->m_type; } + void recordCallSite(ULONG instrOffset, + CORINFO_SIG_INFO *callSig, + CORINFO_METHOD_HANDLE methodHandle) override { + + } + }; #endif From d940e32713b818dd81be5af809525f4e372fe422 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 15:23:53 +1000 Subject: [PATCH 052/273] Windows support in progress --- CMakeLists.txt | 18 +++++++++++------- Pyjion/cee.h | 1 - Pyjion/ilgen.h | 1 - Pyjion/pycomp.cpp | 3 --- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0db4ddb09..31e6d78ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,14 +7,12 @@ set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) -find_package (Python3 COMPONENTS Interpreter Development) +find_package (Python3 3.9 COMPONENTS Interpreter Development) include_directories(${Python3_INCLUDE_DIRS}) set (CLR_DIR CoreCLR/src/coreclr) -add_compile_options(-fexceptions) add_definitions(-DUSE_STL) -add_compile_options(-fvisibility=hidden) IF(CMAKE_BUILD_TYPE MATCHES Debug) message(STATUS "Enabling very verbose messages") @@ -25,6 +23,9 @@ if(NOT WIN32) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) add_compile_options(-fdeclspec) + add_compile_options(-fexceptions) + add_compile_options(-fvisibility=hidden) + add_definitions(-DTARGET_UNIX) message(STATUS "Enabling UNIX Patches") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -32,6 +33,10 @@ if(NOT WIN32) else(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-conversion-null -Wno-pointer-arith) endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") +else() + if (MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++latest") + endif(MSVC) endif(NOT WIN32) @@ -67,12 +72,12 @@ endif() include_directories(CoreCLR/src/coreclr/src/inc CoreCLR/src/coreclr/src/jit) if (WIN32) - set(CLR_OS_BUILD WIN.x64.Debug) + set(CLR_OS_BUILD Windows_NT.x64.Debug) set(CLR_JIT_LIB "clrjit.dll") endif() if (LINUX) - set(CLR_OS_BUILD OSX.x64.Debug) # todo - find out what this is called + set(CLR_OS_BUILD Linux.x64.Debug) set(CLR_JIT_LIB "libclrjit.so") endif() @@ -94,8 +99,7 @@ set(SOURCES Pyjion/absint.cpp Pyjion/absvalue.cpp Pyjion/intrins.cpp Pyjion/jiti add_library(pyjion MODULE ${SOURCES}) install(TARGETS pyjion - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + LIBRARY DESTINATION ${CMAKE_BINARY_DIR}) set_target_properties( pyjion diff --git a/Pyjion/cee.h b/Pyjion/cee.h index b27be06d3..08023a12b 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -45,7 +45,6 @@ #include #include -#include #include #include #include diff --git a/Pyjion/ilgen.h b/Pyjion/ilgen.h index 58703d6ab..61e2b1525 100644 --- a/Pyjion/ilgen.h +++ b/Pyjion/ilgen.h @@ -44,7 +44,6 @@ #include #include -#include #include #include diff --git a/Pyjion/pycomp.cpp b/Pyjion/pycomp.cpp index 5d2ec4e98..ed4990d39 100644 --- a/Pyjion/pycomp.cpp +++ b/Pyjion/pycomp.cpp @@ -35,9 +35,6 @@ CCorJitHost g_jitHost; void CeeInit() { jitStartup(&g_jitHost); -#if _DEBUG - DisableThrowCheck(); -#endif } class InitHolder { From c490efd21a26e1af2fc8185a28bdc7c6ab5a508f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 19:58:45 +1000 Subject: [PATCH 053/273] Start WITH_EXCEPT_START implementation, guess it would be similar to SETUP_EXCEPT that used to exist in the old codebase --- Pyjion/absint.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index fcf28b496..c6609d84b 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -748,7 +748,7 @@ bool AbstractInterpreter::interpret() { } case POP_BLOCK: // Restore the stack state to what we had on entry - lastState.m_stack = m_startStates[m_blockStarts[opcodeIndex]].m_stack; + //lastState.m_stack = m_startStates[m_blockStarts[opcodeIndex]].m_stack; // TODO : Find out what this did, its commented out but it looks important?! //merge_states(m_startStates[m_blockStarts[opcodeIndex]], lastState); break; @@ -842,6 +842,25 @@ bool AbstractInterpreter::interpret() { lastState.pop(); lastState.push(&Bool); break; + case WITH_EXCEPT_START: { + /* At the top of the stack are 7 values: + - (TOP, SECOND, THIRD) = exc_info() + - (FOURTH, FIFTH, SIXTH) = previous exception for EXCEPT_HANDLER + - SEVENTH: the context.__exit__ bound method + We call SEVENTH(TOP, SECOND, THIRD). + Then we push again the TOP exception and the __exit__ + return value. + */ + auto top = lastState.pop_no_escape(); // exc + auto second = lastState.pop_no_escape(); // val + auto third = lastState.pop_no_escape(); // tb + auto seventh = lastState[lastState.stack_size() - 7]; // exit_func + // TODO : Vectorcall (exit_func, stack+1, 3, ..) + PyErr_Format(PyExc_ValueError, + "VectorCalls not implemented"); + lastState.push(&Any); // res + break; + } default: PyErr_Format(PyExc_ValueError, "Unknown unsupported opcode: %s", opcode_name(opcode)); From 0388f537ebff5c5c8ddf5511f84d07faab174399 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 1 Oct 2020 20:11:56 +1000 Subject: [PATCH 054/273] Windows C++20 compat --- CMakeLists.txt | 7 ++++--- CMakeSettings.json | 16 ++++++++++++++++ Pyjion/cee.h | 9 +++++++++ Pyjion/cowvector.h | 2 +- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 CMakeSettings.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 31e6d78ae..4c9a5a8e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,6 @@ project(pyjion) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) find_package (Python3 3.9 COMPONENTS Interpreter Development) include_directories(${Python3_INCLUDE_DIRS}) @@ -20,6 +18,8 @@ IF(CMAKE_BUILD_TYPE MATCHES Debug) ENDIF(CMAKE_BUILD_TYPE MATCHES Debug) if(NOT WIN32) + set(CMAKE_CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD_REQUIRED ON) include_directories(CoreCLR/src/coreclr/src/pal/inc/rt CoreCLR/src/coreclr/src/pal/inc CoreCLR/src/coreclr/src/pal/prebuilt/inc) add_compile_options(-DPAL_STDCPP_COMPAT) add_compile_options(-fdeclspec) @@ -34,6 +34,7 @@ if(NOT WIN32) add_compile_options(-Wno-conversion-null -Wno-pointer-arith) endif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") else() + add_definitions(-DWINDOWS) if (MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++latest") endif(MSVC) @@ -73,7 +74,7 @@ include_directories(CoreCLR/src/coreclr/src/inc CoreCLR/src/coreclr/src/jit) if (WIN32) set(CLR_OS_BUILD Windows_NT.x64.Debug) - set(CLR_JIT_LIB "clrjit.dll") + set(CLR_JIT_LIB "lib/coreclr_static.lib") endif() if (LINUX) diff --git a/CMakeSettings.json b/CMakeSettings.json new file mode 100644 index 000000000..4e332c38b --- /dev/null +++ b/CMakeSettings.json @@ -0,0 +1,16 @@ +{ + // See https://go.microsoft.com/fwlink/?linkid=834763 for more information about this file. + "configurations": [ + { + "name": "x64-Debug", + "generator": "Ninja", + "configurationType": "Debug", + "inheritEnvironments": [ "msvc_x64_x64" ], + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "", + "buildCommandArgs": "", + "ctestCommandArgs": "" + } + ] +} \ No newline at end of file diff --git a/Pyjion/cee.h b/Pyjion/cee.h index 08023a12b..c42bcc585 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -55,16 +55,25 @@ using namespace std; class CCorJitHost : public ICorJitHost { protected: +#ifdef WINDOWS + map intSettings; + map strSettings; +#else map intSettings; map strSettings; +#endif public: CCorJitHost(){ // DEBUG settings. +#ifdef WINDOWS + // TODO: Figure out how to do this in windows! +#else intSettings[u"JitLsraStats"] = 1; intSettings[u"DumpJittedMethods"] = 1; intSettings[u"JitDumpToDebugger"] = 1; intSettings[u"JitDumpASCII"] = 1; strSettings[u"JitDump"] = u"methodname"; +#endif } void * allocateMemory(size_t size) override diff --git a/Pyjion/cowvector.h b/Pyjion/cowvector.h index 3da3f2f6a..5727ef8bb 100644 --- a/Pyjion/cowvector.h +++ b/Pyjion/cowvector.h @@ -55,7 +55,7 @@ template class CowData { protected: // Returns an instance of the data which isn't shared and is safe to mutate. T & get_mutable() { - if (!m_data.unique()) { + if (m_data.use_count() != 1) { m_data = shared_ptr(new T(*m_data)); } return *m_data; From 30d84231097eb8c16ae419c48b1729c73f8eba5a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 2 Oct 2020 06:53:20 +1000 Subject: [PATCH 055/273] Use a better string lookup type. --- Pyjion/cee.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Pyjion/cee.h b/Pyjion/cee.h index c42bcc585..b74f4a9e2 100644 --- a/Pyjion/cee.h +++ b/Pyjion/cee.h @@ -55,13 +55,8 @@ using namespace std; class CCorJitHost : public ICorJitHost { protected: -#ifdef WINDOWS - map intSettings; - map strSettings; -#else - map intSettings; - map strSettings; -#endif + map intSettings; + map strSettings; public: CCorJitHost(){ // DEBUG settings. From 0b2f515b48361b3675fe201e57ea8415fe05b65a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 2 Oct 2020 07:58:46 +1000 Subject: [PATCH 056/273] Big linting and code cleanup to try and squash bugs --- CMakeLists.txt | 1 + Pyjion/absint.cpp | 29 ++++----- Pyjion/absint.h | 43 +++++-------- Pyjion/absvalue.h | 158 ++++++++++++++++++++-------------------------- Pyjion/cee.h | 9 +-- Pyjion/pyjit.cpp | 30 +++++---- 6 files changed, 119 insertions(+), 151 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c9a5a8e7..f2da57b44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ if(NOT WIN32) add_definitions(-DTARGET_UNIX) message(STATUS "Enabling UNIX Patches") + add_compile_options(-Wswitch) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wno-null-arithmetic) else(CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/Pyjion/absint.cpp b/Pyjion/absint.cpp index c6609d84b..0957ac8d0 100644 --- a/Pyjion/absint.cpp +++ b/Pyjion/absint.cpp @@ -36,11 +36,12 @@ #define NUM_ARGS(n) ((n)&0xFF) #define NUM_KW_ARGS(n) (((n)>>8) & 0xff) -#define GET_OPARG(index) _Py_OPARG(m_byteCode[index/sizeof(_Py_CODEUNIT)]) -#define GET_OPCODE(index) _Py_OPCODE(m_byteCode[index/sizeof(_Py_CODEUNIT)]) +#define GET_OPARG(index) _Py_OPARG(m_byteCode[(index)/sizeof(_Py_CODEUNIT)]) +#define GET_OPCODE(index) _Py_OPCODE(m_byteCode[(index)/sizeof(_Py_CODEUNIT)]) AbstractInterpreter::AbstractInterpreter(PyCodeObject *code, IPythonCompiler* comp) : m_code(code), m_comp(comp) { + // TODO : Initialize m_blockIds m_byteCode = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code); m_size = PyBytes_Size(code->co_code); m_returnValue = &Undefined; @@ -869,7 +870,7 @@ bool AbstractInterpreter::interpret() { update_start_state(lastState, curByte + sizeof(_Py_CODEUNIT)); } next:; - } while (queue.size() != 0); + } while (!queue.empty()); return true; } @@ -1091,8 +1092,8 @@ void AbstractInterpreter::dump() { printf(" %-3Id %-22s %d (%s)\r\n", byteIndex, opcode_name(opcode), - oparg, - PyUnicode_AsUTF8(PyTuple_GetItem(m_code->co_names, oparg)) + oparg + //PyUnicode_AsUTF8(PyTuple_GetItem(m_code->co_names, oparg)) ); break; case LOAD_CONST: @@ -1186,7 +1187,7 @@ void AbstractInterpreter::dump_sources(AbstractSource* sources) { } } -char* AbstractInterpreter::opcode_name(int opcode) { +const char* AbstractInterpreter::opcode_name(int opcode) { #define OP_TO_STR(x) case x: return #x; switch (opcode) { OP_TO_STR(POP_TOP) @@ -1373,7 +1374,7 @@ AbstractSource* AbstractInterpreter::add_intermediate_source(size_t opcodeIndex) // Checks to see if we have a non-zero error code on the stack, and if so, // branches to the current error handler. Consumes the error code in the process -void AbstractInterpreter::int_error_check(char* reason) { +void AbstractInterpreter::int_error_check(const char* reason) { auto noErr = m_comp->emit_define_label(); m_comp->emit_int(0); m_comp->emit_branch(BranchEqual, noErr); @@ -1384,7 +1385,7 @@ void AbstractInterpreter::int_error_check(char* reason) { // Checks to see if we have a null value as the last value on our stack // indicating an error, and if so, branches to our current error handler. -void AbstractInterpreter::error_check(char *reason) { +void AbstractInterpreter::error_check(const char *reason) { auto noErr = m_comp->emit_define_label(); m_comp->emit_dup(); m_comp->emit_store_local(m_errorCheckLocal); @@ -1426,7 +1427,7 @@ vector