diff --git a/src/datatype.c b/src/datatype.c index b225ff3bd4fe24..3fc37f199bfd82 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -826,6 +826,22 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name, return bt; } +JL_DLLEXPORT int jl_reinit_foreign_type(jl_datatype_t *dt, + jl_markfunc_t markfunc, + jl_sweepfunc_t sweepfunc) +{ + if (!jl_is_foreign_type(dt)) + return 0; + const jl_datatype_layout_t *layout = dt->layout; + jl_fielddescdyn_t * desc = + (jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout)); + assert(!desc->markfunc); + assert(!desc->sweepfunc); + desc->markfunc = markfunc; + desc->sweepfunc = sweepfunc; + return 1; +} + JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt) { return jl_is_datatype(dt) && dt->layout && dt->layout->fielddesc_type == 3; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index f97c9894238591..2e5df94dc72007 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -343,6 +343,7 @@ XX(jl_new_code_info_uninit) \ XX(jl_new_datatype) \ XX(jl_new_foreign_type) \ + XX(jl_reinit_foreign_type) \ XX(jl_new_method_instance_uninit) \ XX(jl_new_method_table) \ XX(jl_new_method_uninit) \ diff --git a/src/julia_gcext.h b/src/julia_gcext.h index 669e80d069fa40..27f0a6b5ec11c4 100644 --- a/src/julia_gcext.h +++ b/src/julia_gcext.h @@ -49,6 +49,13 @@ JL_DLLEXPORT jl_datatype_t *jl_new_foreign_type( int haspointers, int large); + +#define HAVE_JL_REINIT_FOREIGN_TYPE 1 +JL_DLLEXPORT int jl_reinit_foreign_type( + jl_datatype_t *dt, + jl_markfunc_t markfunc, + jl_sweepfunc_t sweepfunc); + JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt); JL_DLLEXPORT size_t jl_gc_max_internal_obj_size(void); diff --git a/src/staticdata.c b/src/staticdata.c index e1f0f86aa68fc3..7b33db4eadccc2 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -77,6 +77,7 @@ External links: #include "julia.h" #include "julia_internal.h" +#include "julia_gcext.h" #include "builtin_proto.h" #include "processor.h" #include "serialize.h" @@ -1249,6 +1250,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED ios_write(s->s, (char*)v, sizeof(void*) + jl_string_len(v)); write_uint8(s->s, '\0'); // null-terminated strings for easier C-compatibility } + else if (jl_is_foreign_type(t) == 1) { + jl_error("Cannot serialize instances of foreign datatypes"); + } else if (jl_datatype_nfields(t) == 0) { // The object has no fields, so we just snapshot its byte representation assert(!t->layout->npointers); @@ -1438,10 +1442,14 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (dt->layout != NULL) { size_t nf = dt->layout->nfields; size_t np = dt->layout->npointers; - size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); + size_t fieldsize = 0; + uint8_t is_foreign_type = dt->layout->fielddesc_type == 3; + if (!is_foreign_type) { + fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); + } char *flddesc = (char*)dt->layout; size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize; - if (dt->layout->first_ptr != -1) + if (!is_foreign_type && dt->layout->first_ptr != -1) fldsize += np << dt->layout->fielddesc_type; uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*)); write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream @@ -1450,6 +1458,13 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_datatype_t, layout))); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + layout)); // relocation target ios_write(s->const_data, flddesc, fldsize); + if (is_foreign_type) { + // make sure we have space for the extra hidden pointers + // zero them since they will need to be re-initialized externally + assert(fldsize == sizeof(jl_datatype_layout_t)); + jl_fielddescdyn_t dyn = {0, 0}; + ios_write(s->const_data, (char*)&dyn, sizeof(jl_fielddescdyn_t)); + } } } else if (jl_is_typename(v)) { diff --git a/test/gcext/.gitignore b/test/gcext/.gitignore index 0f8c848e5cea69..829c3297dfa2c5 100644 --- a/test/gcext/.gitignore +++ b/test/gcext/.gitignore @@ -1,2 +1,3 @@ /gcext /gcext-debug +/Foreign/deps diff --git a/test/gcext/DependsOnForeign/Manifest.toml b/test/gcext/DependsOnForeign/Manifest.toml new file mode 100644 index 00000000000000..d830116bb54caf --- /dev/null +++ b/test/gcext/DependsOnForeign/Manifest.toml @@ -0,0 +1,14 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.8.3" +manifest_format = "2.0" +project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb" + +[[deps.Foreign]] +deps = ["Libdl"] +path = "../Foreign" +uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4" +version = "0.1.0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/gcext/DependsOnForeign/Project.toml b/test/gcext/DependsOnForeign/Project.toml new file mode 100644 index 00000000000000..b2bee1338c2b76 --- /dev/null +++ b/test/gcext/DependsOnForeign/Project.toml @@ -0,0 +1,6 @@ +name = "DependsOnForeign" +uuid = "4b0716e0-dfb5-4e00-8b44-e2685a41517f" +version = "0.1.0" + +[deps] +Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4" diff --git a/test/gcext/DependsOnForeign/src/DependsOnForeign.jl b/test/gcext/DependsOnForeign/src/DependsOnForeign.jl new file mode 100644 index 00000000000000..cdf31774956e1d --- /dev/null +++ b/test/gcext/DependsOnForeign/src/DependsOnForeign.jl @@ -0,0 +1,14 @@ +module DependsOnForeign + +using Foreign + +f(obj::FObj) = Base.pointer_from_objref(obj) +precompile(f, (FObj,)) + +const FObjRef = Ref{FObj}() + +function __init__() + FObjRef[] = FObj() +end + +end # module DependsOnForeign diff --git a/test/gcext/Foreign/Manifest.toml b/test/gcext/Foreign/Manifest.toml new file mode 100644 index 00000000000000..25cf111aa50ba7 --- /dev/null +++ b/test/gcext/Foreign/Manifest.toml @@ -0,0 +1,8 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.0-DEV" +manifest_format = "2.0" +project_hash = "7b70172a2edbdc772ed789e79d4411d7528eae86" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/gcext/Foreign/Project.toml b/test/gcext/Foreign/Project.toml new file mode 100644 index 00000000000000..819f64beee442c --- /dev/null +++ b/test/gcext/Foreign/Project.toml @@ -0,0 +1,6 @@ +name = "Foreign" +uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4" +version = "0.1.0" + +[deps] +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/gcext/Foreign/deps/foreignlib.c b/test/gcext/Foreign/deps/foreignlib.c new file mode 100644 index 00000000000000..72e02e9bef0cf6 --- /dev/null +++ b/test/gcext/Foreign/deps/foreignlib.c @@ -0,0 +1,56 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "julia.h" +#include "julia_gcext.h" + +// TODO make these atomics +int nmarks = 0; +int nsweeps = 0; + +uintptr_t mark(jl_ptls_t ptls, jl_value_t *p) +{ + nmarks += 1; + return 0; +} + +void sweep(jl_value_t *p) +{ + nsweeps++; +} + +JL_DLLEXPORT jl_datatype_t *declare_foreign(jl_sym_t* name, jl_module_t *module, jl_datatype_t *parent) +{ + return jl_new_foreign_type(name, module, parent, mark, sweep, 1, 0); +} + +// #define GC_MAX_SZCLASS (2032 - sizeof(void *)) + +JL_DLLEXPORT int reinit_foreign(jl_datatype_t *dt) +{ + int ret = jl_reinit_foreign_type(dt, mark, sweep); + nmarks = nsweeps = 0; + if (ret == 0) + return 0; + if (dt->layout->npointers != 1) + return -1; + if (dt->layout->size != 0) + return -2; + return ret; +} + +JL_DLLEXPORT jl_value_t *allocate_foreign(jl_ptls_t ptls, size_t sz, jl_datatype_t *dt) +{ + jl_value_t* obj = jl_gc_alloc_typed(ptls, sz, dt); + jl_gc_schedule_foreign_sweepfunc(ptls, obj); + return obj; +} + +JL_DLLEXPORT int nmark_counter() +{ + return nmarks; +} + +JL_DLLEXPORT int nsweep_counter() +{ + return nsweeps; +} diff --git a/test/gcext/Foreign/src/Foreign.jl b/test/gcext/Foreign/src/Foreign.jl new file mode 100644 index 00000000000000..aaaa71ef98edc9 --- /dev/null +++ b/test/gcext/Foreign/src/Foreign.jl @@ -0,0 +1,29 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Foreign + +using Libdl + +const foreignlib = joinpath(dirname(joinpath(@__DIR__)), "deps", "foreignlib.$(dlext)") + +const FObj = ccall((:declare_foreign, foreignlib), Any, (Any, Any, Any), :FObj, @__MODULE__, Any) +FObj() = ccall((:allocate_foreign, foreignlib), Any, (Ptr{Cvoid}, Csize_t, Any,), Core.getptls(), sizeof(Ptr{Cvoid}), FObj)::FObj + +export FObj + +get_nmark() = ccall((:nmark_counter, foreignlib), Cint, ()) +get_nsweep() = ccall((:nsweep_counter, foreignlib), Cint, ()) + +function __init__() + @assert ccall((:reinit_foreign, foreignlib), Cint, (Any,), FObj) == 1 +end + +allocs(N) = [Foreign.FObj() for _ in 1:N] + +function test(N) + x = allocs(N) + Core.donotdelete(x) + x = nothing +end + +end # module Foreign diff --git a/test/gcext/ForeignObjSerialization/Manifest.toml b/test/gcext/ForeignObjSerialization/Manifest.toml new file mode 100644 index 00000000000000..d830116bb54caf --- /dev/null +++ b/test/gcext/ForeignObjSerialization/Manifest.toml @@ -0,0 +1,14 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.8.3" +manifest_format = "2.0" +project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb" + +[[deps.Foreign]] +deps = ["Libdl"] +path = "../Foreign" +uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4" +version = "0.1.0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/gcext/ForeignObjSerialization/Project.toml b/test/gcext/ForeignObjSerialization/Project.toml new file mode 100644 index 00000000000000..1a26ff7884481c --- /dev/null +++ b/test/gcext/ForeignObjSerialization/Project.toml @@ -0,0 +1,6 @@ +name = "ForeignObjSerialization" +uuid = "2c015d96-a6ca-42f0-bc68-f9090de6bc2c" +version = "0.1.0" + +[deps] +Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4" diff --git a/test/gcext/ForeignObjSerialization/src/ForeignObjSerialization.jl b/test/gcext/ForeignObjSerialization/src/ForeignObjSerialization.jl new file mode 100644 index 00000000000000..e32753aecb3b4c --- /dev/null +++ b/test/gcext/ForeignObjSerialization/src/ForeignObjSerialization.jl @@ -0,0 +1,6 @@ +module ForeignObjSerialization + +using Foreign +const FObjRef = Ref{FObj}(FObj()) + +end # module ForeignObjSerialization diff --git a/test/gcext/Makefile b/test/gcext/Makefile index b3314d1f9b32b8..2dcac23aaa858e 100644 --- a/test/gcext/Makefile +++ b/test/gcext/Makefile @@ -19,18 +19,26 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) # get the executable suffix, if any EXE := $(suffix $(abspath $(JULIA))) +OS := $(shell uname) +ifeq ($(OS), Darwin) + DYLIB := .dylib +else + DYLIB := .so +endif + # get compiler and linker flags. (see: `contrib/julia-config.jl`) JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' -- CPPFLAGS_ADD := CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) +DYLIBFLAGS := --shared -fPIC DEBUGFLAGS += -g #============================================================================= -release: $(BIN)/gcext$(EXE) -debug: $(BIN)/gcext-debug$(EXE) +release: $(BIN)/gcext$(EXE) $(BIN)/Foreign/deps/foreignlib$(DYLIB) +debug: $(BIN)/gcext-debug$(EXE) $(BIN)/Foreign/deps/foreignlib-debug$(DYLIB) $(BIN)/gcext$(EXE): $(SRCDIR)/gcext.c $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) @@ -38,6 +46,12 @@ $(BIN)/gcext$(EXE): $(SRCDIR)/gcext.c $(BIN)/gcext-debug$(EXE): $(SRCDIR)/gcext.c $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS) +$(BIN)/Foreign/deps/foreignlib$(DYLIB): $(SRCDIR)/Foreign/deps/foreignlib.c + $(CC) $^ -o $@ $(DYLIBFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) + +$(BIN)/Foreign/deps/foreignlib-debug$(DYLIB): $(SRCDIR)/Foreign/deps/foreignlib.c + $(CC) $^ -o $@ $(DYLIBFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS) + ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR))) # for demonstration purposes, our demo code is also installed # in $BIN, although this would likely not be typical @@ -45,12 +59,14 @@ $(BIN)/LocalTest.jl: $(SRCDIR)/LocalTest.jl cp $< $@ endif -check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl +check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl $(BIN)/Foreign/deps/foreignlib$(DYLIB) $(JULIA) --depwarn=error $(SRCDIR)/gcext-test.jl $< @echo SUCCESS clean: -rm -f $(BIN)/gcext-debug$(EXE) $(BIN)/gcext$(EXE) + -rm -f $(BIN)/Foreign/deps/foreignlib$(DYLIB) + -rm -f $(BIN)/Foreign/deps/foreignlib-debug$(DYLIB) .PHONY: release debug clean check diff --git a/test/gcext/gcext-test.jl b/test/gcext/gcext-test.jl index 0dc9bbadd92b52..81637392e3c5d5 100644 --- a/test/gcext/gcext-test.jl +++ b/test/gcext/gcext-test.jl @@ -2,6 +2,7 @@ # tests the output of the embedding example is correct using Test +using Pkg if Sys.iswindows() # libjulia needs to be in the same directory as the embedding executable or in path @@ -43,3 +44,35 @@ end @test checknum(lines[5], r"([0-9]+) corrupted auxiliary roots", n -> n == 0) end + +@testset "Package with foreign type" begin + load_path = copy(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "Foreign")) + push!(LOAD_PATH, joinpath(@__DIR__, "DependsOnForeign")) + try + # Force recaching + Base.compilecache(Base.identify_package("Foreign")) + Base.compilecache(Base.identify_package("DependsOnForeign")) + + push!(LOAD_PATH, joinpath(@__DIR__, "ForeignObjSerialization")) + @test_throws ErrorException Base.compilecache(Base.identify_package("ForeignObjSerialization"), Base.DevNull()) + pop!(LOAD_PATH) + + (@eval (using Foreign)) + @test Base.invokelatest(Foreign.get_nmark) == 0 + @test Base.invokelatest(Foreign.get_nsweep) == 0 + + obj = Base.invokelatest(Foreign.FObj) + GC.@preserve obj begin + GC.gc(true) + end + @test Base.invokelatest(Foreign.get_nmark) > 0 + @time Base.invokelatest(Foreign.test, 10) + GC.gc(true) + @test Base.invokelatest(Foreign.get_nsweep) > 0 + (@eval (using DependsOnForeign)) + Base.invokelatest(DependsOnForeign.f, obj) + finally + copy!(LOAD_PATH, load_path) + end +end