From 7dbf616ecfe78b1075dea5815e8493db67b2679d Mon Sep 17 00:00:00 2001 From: SimonDanisch Date: Sat, 17 Feb 2018 12:27:50 +0100 Subject: [PATCH 1/2] wrappers for other languages --- .gitignore | 1 + src/PackageCompiler.jl | 1 + src/shared_library.jl | 137 +++++++++++++++++++++++++++++++++++++++++ test/pytest/pymodule.c | 22 +++++++ test/pytest/pymodule.h | 1 + test/pytest/pytest.jl | 130 ++++++++++++++++++++++++++++++++++++++ test/pytest/setup.py | 12 ++++ test/runtests.jl | 49 +++++++++++++++ 8 files changed, 353 insertions(+) create mode 100644 src/shared_library.jl create mode 100644 test/pytest/pymodule.c create mode 100644 test/pytest/pymodule.h create mode 100644 test/pytest/pytest.jl create mode 100644 test/pytest/setup.py diff --git a/.gitignore b/.gitignore index b8c2c05f..f631e006 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ sysimg *.dll hello hello.exe +test/pytest/build/ diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 51586847..48217826 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -21,6 +21,7 @@ iswindows() && using WinRPM include("static_julia.jl") include("api.jl") include("snooping.jl") +include("shared_library.jl") const sysimage_binaries = ( "sys.o", "sys.$(Libdl.dlext)", "sys.ji", "inference.o", "inference.ji" diff --git a/src/shared_library.jl b/src/shared_library.jl new file mode 100644 index 00000000..72952a89 --- /dev/null +++ b/src/shared_library.jl @@ -0,0 +1,137 @@ + +function extract_func(expr::Expr) + signature = expr.args[2] + functype = signature.args[2] + if functype.head == :call && functype.args[1] == :typeof + return functype.args[2].args[2].value, signature.args[3:end] # typeof(Module.$(QuoteNode(func))) + else # only real functions, no getfield functions will be exported + Symbol(""), signature.args[3:end] + end +end + +function to_ctype(T) + T == UInt8 && return "unsigned char" + T == Bool && return "bool" + T == Int16 && return "short" + T == UInt16 && return "unsigned short" + T == Int32 && return "int" + T == UInt32 && return "unsigned int" + T == Int64 && return "long long" + T == UInt64 && return "unsigned long long" + T == Int64 && return "intmax_t" + T == UInt64 && return "uintmax_t" + T == Float32 && return "float" + T == Float64 && return "double" + T == Complex{Float32} && return "complex float" + T == Complex{Float64} && return "complex double" + T == Int && return "ptrdiff_t" + T == Int && return "ssize_t" + T == UInt && return "size_t" + T == Char && return "int" + T == Void && return "void" + T == Union{} && return "void " + T == Ptr{Void} && return "void*" + (T <: Ptr{T} where T) && return string(to_ctype(eltype(T)), "*") + T == Ptr{Ptr{UInt8}} && return "char** " + T == Ref{Any} && return "jl_value_t** " + return "jl_value_t* /*$T*/ " +end + + +exports_function(mod, name) = name in names(mod) # + +function unique_method_name(fname, used = Dict{Symbol, Int}()) + i = get!(used, fname, 0) + used[fname] += 1 + Symbol(string(fname, i == 0 ? "" : i)) # only append number if i != 0 +end + + +function write_ccallable(jl_io, c_io, mod, func, argtypes, used) + argnames = map(x-> Symbol("arg_$(x)"), 1:length(argtypes)) + args = map(n_t-> Expr(:(::), n_t...), zip(argnames, argtypes)) + realfunc = getfield(mod, func) + types = eval.(argtypes) + returntype = Core.Inference.return_type(realfunc, types) + method = unique_method_name(func, used) + expr = """ + Base.@ccallable function $(method)($(join(args, ", ")))::$(returntype) + $(mod).$(func)($(join(argnames, ", "))) + end + """ + println(jl_io, expr) + ctypes = to_ctype.(types) + # cargs = map(t_n-> join(n_t, " "), zip(argtypes, argnames)) + expr = """ + extern $(to_ctype(returntype)) $(method)($(join(ctypes, ", "))); + """ + println(c_io, expr) +end + +function emit_shared_julia(folder, mod, snoopfile, name = lowercase(string(mod))) + open(joinpath(folder, "snoopy.jl"), "w") do io + println(io, "include(\"$(escape_string(snoopfile))\")") + end + csv = joinpath(folder, "snooped.csv") + cd(folder) do + SnoopCompile.@snoop csv begin + include("snoopy.jl") + end + end + data = SnoopCompile.read(csv) + pc = SnoopCompile.parcel(reverse!(data[2])) + fname = joinpath(folder, name) + open(fname * ".jl", "w") do jl_io + println(jl_io, "module C_$(mod)") + println(jl_io, "import $(mod)") + open(fname * ".h", "w") do c_io + println(c_io, """ + // Standard headers + #include + #include + // Julia headers (for initialization and gc commands) + #include "uv.h" + #include "julia.h" + """) + used = Dict{Symbol, Int}() + for (k, v) in pc + for ln in v + # replace `_` for free parameters, which print out a warning otherwise + expr = parse(ln) # parse to make sure expression is parsing without error + funcsym, argtypes = extract_func(expr) + if exports_function(mod, funcsym) + write_ccallable(jl_io, c_io, mod, funcsym, argtypes, used) + end + end + end + println(jl_io, "end") + end + end + fname * ".jl", fname * ".h" +end + +function get_module(mod::Symbol) + eval(Main, :(using $mod)) + getfield(Main, mod) +end + +function compile_sharedlib( + folder, package_name::Symbol, + snoopfile = Pkg.dir(string(package_name), "test", "runtests.jl") + ) + mod = get_module(package_name) + isdir(folder) || mkdir(folder) + name = lowercase(string(package_name)) + shared_jl, shared_h = emit_shared_julia(folder, mod, snoopfile, name) + builddir = joinpath(folder, "build") + PackageCompiler.julia_compile( + shared_jl; + julia_program_basename = name, + verbose = true, quiet = false, object = true, + sysimage = nothing, cprog = nothing, builddir = builddir, + cpu_target = nothing, optimize = nothing, debug = nothing, + inline = nothing, check_bounds = nothing, math_mode = nothing, + executable = false, shared = true, julialibs = true + ) + joinpath(builddir, name * ".$(Libdl.dlext)"), shared_h +end diff --git a/test/pytest/pymodule.c b/test/pytest/pymodule.c new file mode 100644 index 00000000..ea99c14e --- /dev/null +++ b/test/pytest/pymodule.c @@ -0,0 +1,22 @@ +#include + +static PyObject* say_hello(PyObject* self, PyObject* args) +{ + const char* name; + + if (!PyArg_ParseTuple(args, "s", &name)) + return NULL; + + printf("Hello %s!\n", name); + + Py_RETURN_NONE; +} + +static PyMethodDef helloworld_funcs[] = { + {"say_hello", (PyCFunction)say_hello, METH_VARARGS, "say_hello( ): Any message you want to put here!!\n"}, + {NULL} +}; + +void inithello(void) { + Py_InitModule3("hello", helloworld_funcs, "nice stuf"); +} diff --git a/test/pytest/pymodule.h b/test/pytest/pymodule.h new file mode 100644 index 00000000..8b8161f2 --- /dev/null +++ b/test/pytest/pymodule.h @@ -0,0 +1 @@ +PyObject* helloworld(PyObject*); diff --git a/test/pytest/pytest.jl b/test/pytest/pytest.jl new file mode 100644 index 00000000..5934d4ee --- /dev/null +++ b/test/pytest/pytest.jl @@ -0,0 +1,130 @@ +using PackageCompiler + +PackageCompiler.julia_compile( + joinpath(@__DIR__, "pyshared.jl"); + julia_program_basename = "pyshared", + verbose = true, quiet = false, object = true, + sysimage = nothing, cprog = nothing, builddir = @__DIR__, + cpu_target = nothing, optimize = nothing, debug = nothing, + inline = nothing, check_bounds = nothing, math_mode = nothing, + executable = false, shared = true, julialibs = true +) + +using PackageCompiler +dir(folders...) = abspath(joinpath(homedir(), "UnicodeFun", folders...)) +tmp_dir = dir("build") +o_file = dir("build", "pymodule.o") +cd(dir()) do + PackageCompiler.build_object( + dir("pyshared.jl"), escape_string(tmp_dir), dir("build", "juliamodule.o"), true, + nothing, nothing, nothing, nothing, nothing, nothing, nothing, + nothing, nothing + ) +end +import PackageCompiler: system_compiler, bitness_flag, julia_flags +cc = "gcc"#system_compiler() +bitness = bitness_flag() +flags = julia_flags() +command = `$cc -shared -fPIC -c $(dir("pymodule.c")) -o $(dir("build", "pymodule.o")) ` +command = `$command -IC:\\Python27\\include -IC:\\Python27\\PC` +RPMbindir = PackageCompiler.mingw_dir("bin") +incdir = PackageCompiler.mingw_dir("include") +push!(Base.Libdl.DL_LOAD_PATH, RPMbindir) # TODO does this need to be reversed? +ENV["PATH"] = ENV["PATH"] * ";" * RPMbindir +command = `$command -I$incdir` +try + run(command) +catch e + Base.showerror(STDOUT, e) +end + +command = `$cc $(bitness_flag()) -mdll -O -Wall -IC:\\Python27\\include -IC:\\Python27\\PC -c $(dir("pymodule.c")) -o build\\pymodule.o` +command = `$command -I$incdir` +run(command) +show(dir("build", "juliamodule.o")) +gcc = PackageCompiler.system_compiler() +command = `$gcc $(bitness_flag()) -shared -fPIC -o pymodule.pyd $(dir("build", "pymodule.o")) $(dir("build", "juliamodule.o"))` +command = `$command -IC:\\Python27\\include -IC:\\Python27\\PC` +flags = julia_flags() +incdir = PackageCompiler.mingw_dir("include") +show(incdir) +command = `$command -I$incdir $flags -D MS_WIN64` +run(command) +cd(@__DIR__) +println(incdir) +run(`$gcc -c pymodule.c -IC:\\Python27\\include -I$incdir`) + +gcc -shared hellomodule.o -LC:\Python27\libs -lpython27 -o hello.dll +``` + + + +The problem here is that you said that somewhere you will provide the definition of a class called Rectangle -- where the example code states + +cdef extern from "Rectangle.h" namespace "shapes": + cdef cppclass Rectangle: + ... + +However, when you compiled the library you didn't provide the code for Rectangle, or a library that contained it, so rect.so has no idea where to find this Rectangle class. + +To run your code you must first create the Rectangle object file. + +gcc -c Rectangle.cpp # creates a file called Rectangle.o + +Now you can either create a library to dynamically link against, or statically link the object file into rect.so. I'll cover statically linking first as it's simplest. + +gcc -shared -fPIC -I /usr/include/python2.7 rect.cpp Rectangle.o -o rect.so + +Note that I haven't included the library for python. This is because you expect your library to be loaded by the python interpreter, thus the python libraries will already be loaded by the process when your library is loaded. In addition to providing rect.cpp as a source I also provide Rectangle.o. So lets try running a program using your module. + +run.py + +import rect +print(rect.PyRectangle(0, 0, 1, 2).getLength()) + +Unfortunately, this produces another error: + +ImportError: /home/user/rectangle/rect.so undefined symbol: _ZTINSt8ios_base7failureE + +This is because cython needs the c++ standard library, but python hasn't loaded it. You can fix this by adding the c++ standard library to the required libraries for rect.so + +gcc -shared -fPIC -I/usr/include/python2.7 rect.cpp Rectangle.o -lstdc++ \ + -o rect.so + +Run run.py again and all should work. However, the code for rect.so is larger than it needs to be, especially if you produce multiple libraries that depend on the same code. You can dynamically link the Rectangle code, by making it a library as well. + +gcc -shared -fPIC Rectangle.o -o libRectangle.so +gcc -shared -fPIC -I/usr/include/python2.7 -L. rect.cpp -lRectangle -lstdc++ \ + -o rect.so + +We compile the Rectangle +11 +down vote + + +This worked for me with Python 3.3 : + + create static python lib from dll + + python dll is usually in C:/Windows/System32; in msys shell: + + gendef.exe python33.dll + + dlltool.exe --dllname python33.dll --def python33.def --output-lib libpython33.a + + mv libpython33.a C:/Python33/libs + + use swig to generate wrappers + + e.g., swig -c++ -python myExtension.i + + wrapper MUST be compiled with MS_WIN64, or your computer will crash when you import the class in Python + + g++ -c myExtension.cpp -I/other/includes + + g++ -DMS_WIN64 -c myExtension_wrap.cxx -IC:/Python33/include + + shared library + + g++ -shared -o _myExtension.pyd myExtension.o myExtension_wrap.o -lPython33 -lOtherSharedLibs -LC:/Python33/libs -LC:/path/to/other/shared/libs +``` diff --git a/test/pytest/setup.py b/test/pytest/setup.py new file mode 100644 index 00000000..f2d08c35 --- /dev/null +++ b/test/pytest/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup, Extension +import shutil +import os + +extension = Extension('hello', + sources = ['pymodule.c'], + include_dirs = ['C:\\Users\\sdani\\.julia\\v0.6\\WinRPM\\deps\\usr\\x86_64-w64-mingw32\\sys-root\\mingw\\include'], + language = 'c' +) + +setup(name='hello', version='1.0', \ + ext_modules=[extension]) diff --git a/test/runtests.jl b/test/runtests.jl index c465b01c..df73ea5b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -16,3 +16,52 @@ img_file = PackageCompiler.compile_package("Matcha", "UnicodeFun", force = false @test length(readlines(userimg)) > 700 @test success(`julia -J $(img_file)`) end + +using PackageCompiler +using Base.Test + +@testset "shared library + executable" begin + libdir = mktempdir() + s_file, h_file = PackageCompiler.compile_sharedlib(libdir, :UnicodeFun) + cprog = joinpath(libdir, "test.c") + e_file = joinpath(libdir, "build", "test.exe") + open(cprog, "w") do io + println(io, """ + #include \"$(escape_string(h_file))\" + int main(int argc, char *argv[]){ + intptr_t v; + // Initialize Julia + uv_setup_args(argc, argv); // no-op on Windows + libsupport_init(); + jl_options.image_file = \"$(escape_string(s_file))\"; + julia_init(JL_IMAGE_JULIA_HOME); + // Do some work + int test = to_latex(\\itA \\in \\bbR^{nxn}, \\bfv \\in \\bbR^n, \\lambda_i \\in \\bbR: \\itA\\bfv = \\lambda_i\\bfv"); + printf(\"subscript of 4: %c \\n\", test); + // Cleanup and graceful exit + jl_atexit_hook(0); + return 0; + } + """) + end + PackageCompiler.build_executable(s_file, e_file, cprog, true) + @test success(`test.exe`) + for i = 1:100 + # hm, why is this still needed on windows? Still the rm bug, or is + # @test success(`test`) actually taking quite a bit of time to free the resource? + try + rm(libdir, recursive = true) + end + sleep(1/100) + end +end +libdir = joinpath(homedir(), "UnicodeFun") +s_file, h_file = PackageCompiler.compile_sharedlib(libdir, :UnicodeFun) +write(STDOUT, open(read, h_file)) +cprog = joinpath(libdir, "test.c") +e_file = joinpath(libdir, "build", "test.exe") +cprog = joinpath(libdir, "test.c") +e_file = joinpath(libdir, "build", "test.exe") +s_file = joinpath(libdir, "build", "unicodefun.dll") +PackageCompiler.build_executable(s_file, e_file, cprog, true) +success(`.\\$(e_file)`) From cff563b7f5fec21cffa6588ac04586e751fc6e89 Mon Sep 17 00:00:00 2001 From: SimonDanisch Date: Mon, 19 Feb 2018 11:31:53 +0100 Subject: [PATCH 2/2] try on linux --- test/pytest/pyshared.jl | 6 ++++++ test/pytest/pytest.jl | 12 ------------ 2 files changed, 6 insertions(+), 12 deletions(-) create mode 100644 test/pytest/pyshared.jl diff --git a/test/pytest/pyshared.jl b/test/pytest/pyshared.jl new file mode 100644 index 00000000..7aa9b20d --- /dev/null +++ b/test/pytest/pyshared.jl @@ -0,0 +1,6 @@ +using PyCall + +Base.@ccallable function helloworld(self::PyObject)::PyObject + println(self) + return PyObject("test") +end \ No newline at end of file diff --git a/test/pytest/pytest.jl b/test/pytest/pytest.jl index 5934d4ee..f50753af 100644 --- a/test/pytest/pytest.jl +++ b/test/pytest/pytest.jl @@ -1,15 +1,3 @@ -using PackageCompiler - -PackageCompiler.julia_compile( - joinpath(@__DIR__, "pyshared.jl"); - julia_program_basename = "pyshared", - verbose = true, quiet = false, object = true, - sysimage = nothing, cprog = nothing, builddir = @__DIR__, - cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - executable = false, shared = true, julialibs = true -) - using PackageCompiler dir(folders...) = abspath(joinpath(homedir(), "UnicodeFun", folders...)) tmp_dir = dir("build")