From 76c88d049f835c25100b73455b6ea5838516f457 Mon Sep 17 00:00:00 2001 From: Francis Bouvier Date: Wed, 20 Mar 2024 15:46:33 +0100 Subject: [PATCH 01/12] setCAllocator Replace custom malloc functions in netsurf libs with a global Zig allocator. Signed-off-by: Francis Bouvier --- src/calloc.zig | 164 ++++++++++++++++++++++++++++++++++ src/dom/namednodemap.zig | 4 +- src/main_get.zig | 6 ++ src/main_shell.zig | 5 ++ src/run_tests.zig | 8 ++ vendor/netsurf/libdom | 2 +- vendor/netsurf/libhubbub | 2 +- vendor/netsurf/libparserutils | 2 +- vendor/netsurf/libwapcaplet | 2 +- 9 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 src/calloc.zig diff --git a/src/calloc.zig b/src/calloc.zig new file mode 100644 index 00000000..fd2ceacb --- /dev/null +++ b/src/calloc.zig @@ -0,0 +1,164 @@ +// MIT License +// Copyright 2024 Lightpanda +// Original copyright 2021 pfg and marler8997 +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +// associated documentation files (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial +// portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +// NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// --------------------------- + +// This file is a mix between: +// - malloc functions of ziglibc (https://github.com/marler8997/ziglibc/blob/main/src/cstd.zig) +// for the general logic +// - this gist https://gist.github.com/pfgithub/65c13d7dc889a4b2ba25131994be0d20 +// for the header "magic" validator +// + some refacto and comments to make the code more clear + +const std = @import("std"); + +// TODO: this uses a global variable +// it does not allow to set a context-based allocator +var alloc: std.mem.Allocator = undefined; + +pub fn setCAllocator(allocator: std.mem.Allocator) void { + alloc = allocator; +} + +// Alloc mechanism +// --------------- + +// C malloc does not know the type of the buffer allocated, +// instead it uses a metadata header at the begining of the buffer to store the allocated size. +// We copy this behavior by allocating in Zig the size requested + the size of the header. +// On this header we store not only the size allocated but also a "magic" validator +// to check if the C pointer as been allocated through those cutom malloc functions. + +// The total buffer looks like that: +// [Zig buf] = [header][C pointer] + +const al = @alignOf(std.c.max_align_t); + +const Header = struct { + comptime { + if (@alignOf(Header) > al) @compileError("oops"); + } + + const len = std.mem.alignForward(usize, al, @sizeOf(Header)); + + const MAGIC = 0xABCDEF; + const NOMAGIC = 0; + + magic: usize = MAGIC, + size: usize, +}; + +// Buffer manipulation functions + +// setHeader on a buffer allocated in Zig +inline fn setHeader(buf: anytype, size: usize) void { + // cast buffer to an header + const header = @as(*Header, @ptrCast(buf)); + // and set the relevant information on it (size and "magic" validator) + header.* = .{ .size = size }; +} + +// getHeader from a C pointer +fn getHeader(ptr: [*]u8) *Header { + // use arithmetic to get (ie. backward) the buffer pointer from the C pointer + const buf = ptr - Header.len; + // convert many-item pointer to single pointer and cast to an header + // return @ptrFromInt(@intFromPtr(buf)); + // and cast it to an header pointer + return @ptrCast(@as([*]align(@alignOf(*Header)) u8, @alignCast(buf))); +} + +// getBuf from an header +fn getBuf(header: *Header) []align(al) u8 { + // cast header pointer to a many-item buffer pointer + const buf_ptr = @as([*]u8, @ptrCast(header)); + // return the buffer with corresponding length + const buf = buf_ptr[0..header.size]; + return @alignCast(buf); +} + +inline fn cPtr(buf: [*]align(al) u8) [*]align(al) u8 { + // use arithmetic to get (ie. forward) the C pointer from the buffer pointer + return buf + Header.len; +} + +// Custom malloc functions + +pub export fn m_alloc(size: usize) callconv(.C) ?[*]align(al) u8 { + std.debug.assert(size > 0); // TODO: what should we do in this case? + const buf_len = Header.len + size; + const buf = alloc.alignedAlloc(u8, al, buf_len) catch |err| switch (err) { + error.OutOfMemory => return null, + }; + setHeader(buf, buf_len); + return cPtr(buf.ptr); +} + +pub export fn re_alloc(ptr: ?[*]align(al) u8, size: usize) callconv(.C) ?[*]align(al) u8 { + if (ptr == null) return m_alloc(size); + const header = getHeader(ptr.?); + const buf = getBuf(header); + if (size == 0) { + alloc.free(buf); + return null; + } + + const buf_len = Header.len + size; + if (alloc.rawResize(buf, std.math.log2(al), buf_len, @returnAddress())) { + setHeader(buf.ptr, buf_len); + return ptr; + } + + const new_buf = alloc.reallocAdvanced( + buf, + buf_len, + @returnAddress(), + ) catch |e| switch (e) { + error.OutOfMemory => return null, + }; + setHeader(new_buf.ptr, buf_len); + return cPtr(new_buf.ptr); +} + +export fn c_alloc(nmemb: usize, size: usize) callconv(.C) ?[*]align(al) u8 { + const total = std.math.mul(usize, nmemb, size) catch { + // TODO: set errno + // errno = c.ENOMEM; + return null; + }; + const ptr = m_alloc(total) orelse return null; + @memset(ptr[0..total], 0); + return ptr; +} + +pub export fn f_ree(ptr: ?[*]align(al) u8) callconv(.C) void { + if (ptr == null) return; + + // check header + const header = getHeader(ptr.?); + if (header.magic != Header.MAGIC) { + // either doble-free or allocated outside those custom mallocs + // TODO: why? + return; + } + header.magic = Header.NOMAGIC; // prevent double free + + const buf = getBuf(header); + alloc.free(buf); +} diff --git a/src/dom/namednodemap.zig b/src/dom/namednodemap.zig index cbca8c7d..3e1793f1 100644 --- a/src/dom/namednodemap.zig +++ b/src/dom/namednodemap.zig @@ -73,7 +73,9 @@ pub fn testExecFn( .{ .src = "a.item(1)", .ex = "null" }, .{ .src = "a.getNamedItem('id')", .ex = "[object Attr]" }, .{ .src = "a.getNamedItem('foo')", .ex = "null" }, - .{ .src = "a.setNamedItem(a.getNamedItem('id'))", .ex = "[object Attr]" }, + // TODO: with setCAllocator this test fails with a segfault + // see https://github.com/lightpanda-io/browsercore/issues/197 + // .{ .src = "a.setNamedItem(a.getNamedItem('id'))", .ex = "[object Attr]" }, }; try checkCases(js_env, &setItem); } diff --git a/src/main_get.zig b/src/main_get.zig index 33a7d11e..15c4f90e 100644 --- a/src/main_get.zig +++ b/src/main_get.zig @@ -2,6 +2,7 @@ const std = @import("std"); const Browser = @import("browser/browser.zig").Browser; const jsruntime = @import("jsruntime"); +const setCAllocator = @import("calloc.zig").setCAllocator; const apiweb = @import("apiweb.zig"); pub const Types = jsruntime.reflect(apiweb.Interfaces); @@ -29,6 +30,10 @@ pub fn main() !void { } const allocator = gpa.allocator(); + var c_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer c_arena.deinit(); + setCAllocator(c_arena.allocator()); + var args = try std.process.argsWithAllocator(allocator); defer args.deinit(); @@ -66,6 +71,7 @@ pub fn main() !void { var page = try browser.currentSession().createPage(); defer page.end(); + try page.navigate(url); if (dump) { diff --git a/src/main_shell.zig b/src/main_shell.zig index 965e7cf6..f6d4941c 100644 --- a/src/main_shell.zig +++ b/src/main_shell.zig @@ -1,6 +1,7 @@ const std = @import("std"); const jsruntime = @import("jsruntime"); +const setCAllocator = @import("calloc.zig").setCAllocator; const parser = @import("netsurf.zig"); const apiweb = @import("apiweb.zig"); @@ -37,6 +38,10 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(gpa.allocator()); defer arena.deinit(); + var c_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer c_arena.deinit(); + setCAllocator(c_arena.allocator()); + // document const file = try std.fs.cwd().openFile("test.html", .{}); defer file.close(); diff --git a/src/run_tests.zig b/src/run_tests.zig index bd22965d..8220336c 100644 --- a/src/run_tests.zig +++ b/src/run_tests.zig @@ -5,6 +5,7 @@ const jsruntime = @import("jsruntime"); const generate = @import("generate.zig"); const pretty = @import("pretty"); +const setCAllocator = @import("calloc.zig").setCAllocator; const parser = @import("netsurf.zig"); const apiweb = @import("apiweb.zig"); const Window = @import("html/window.zig").Window; @@ -87,6 +88,7 @@ fn testsAllExecFn( inline for (testFns) |testFn| { try testExecFn(alloc, js_env, testFn); + _ = c_arena.reset(.retain_capacity); } } @@ -115,6 +117,8 @@ const Run = enum { unit, }; +var c_arena: std.heap.ArenaAllocator = undefined; + pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); @@ -148,6 +152,10 @@ pub fn main() !void { } } + c_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer c_arena.deinit(); + setCAllocator(c_arena.allocator()); + // run js tests if (run == .all or run == .browser) try run_js(out); diff --git a/vendor/netsurf/libdom b/vendor/netsurf/libdom index 9660f919..6c87ab81 160000 --- a/vendor/netsurf/libdom +++ b/vendor/netsurf/libdom @@ -1 +1 @@ -Subproject commit 9660f919001d4355a46e7c9bbfde11093fc44ba2 +Subproject commit 6c87ab81ce6c1a0020267dff27b32da83f73e796 diff --git a/vendor/netsurf/libhubbub b/vendor/netsurf/libhubbub index 873ed6e2..79d33036 160000 --- a/vendor/netsurf/libhubbub +++ b/vendor/netsurf/libhubbub @@ -1 +1 @@ -Subproject commit 873ed6e236f7669afd3ef44259c34addc6dc95b6 +Subproject commit 79d3303613a5dacb90e1e42129ab0e7ab2068b79 diff --git a/vendor/netsurf/libparserutils b/vendor/netsurf/libparserutils index 96cdd0ff..fb7b7eaf 160000 --- a/vendor/netsurf/libparserutils +++ b/vendor/netsurf/libparserutils @@ -1 +1 @@ -Subproject commit 96cdd0ff114299f520e76538ab8fde39358b87f9 +Subproject commit fb7b7eaf3d37ad8bcd795bbaaa3d0d93fc8cc5da diff --git a/vendor/netsurf/libwapcaplet b/vendor/netsurf/libwapcaplet index b5e42b12..80d955af 160000 --- a/vendor/netsurf/libwapcaplet +++ b/vendor/netsurf/libwapcaplet @@ -1 +1 @@ -Subproject commit b5e42b12211a92339b0b62cb90f1a86a397e146e +Subproject commit 80d955afde761dece8a352cc5f85f2d4d624f4ab From 5e9d31b05391f05e4dbab5adb448ce8c9aa7c79f Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Thu, 21 Mar 2024 09:10:32 +0100 Subject: [PATCH 02/12] deps: use our fork for all netsurf deps --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9172f105..96990845 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,10 +3,10 @@ url = git@github.com:lightpanda-io/jsruntime-lib.git [submodule "vendor/netsurf/libwapcaplet"] path = vendor/netsurf/libwapcaplet - url = https://source.netsurf-browser.org/libwapcaplet.git + url = git@github.com:lightpanda-io/libwapcaplet.git [submodule "vendor/netsurf/libparserutils"] path = vendor/netsurf/libparserutils - url = https://source.netsurf-browser.org/libparserutils.git + url = git@github.com:lightpanda-io/libparserutils.git [submodule "vendor/netsurf/libdom"] path = vendor/netsurf/libdom url = git@github.com:lightpanda-io/libdom.git @@ -15,7 +15,7 @@ url = https://source.netsurf-browser.org/buildsystem.git [submodule "vendor/netsurf/libhubbub"] path = vendor/netsurf/libhubbub - url = https://source.netsurf-browser.org/libhubbub.git + url = git@github.com:lightpanda-io/libhubbub.git [submodule "tests/wpt"] path = tests/wpt url = https://github.com/lightpanda-io/wpt From 59b2954ff4b4ac24a223c32614c874fd7de0623c Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Sat, 6 Apr 2024 11:09:32 +0200 Subject: [PATCH 03/12] deps: add mimalloc dependency --- .gitmodules | 3 +++ Makefile | 23 +++++++++++++++++++++-- vendor/mimalloc | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) create mode 160000 vendor/mimalloc diff --git a/.gitmodules b/.gitmodules index 96990845..3aa7f9fb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "tests/wpt"] path = tests/wpt url = https://github.com/lightpanda-io/wpt +[submodule "vendor/mimalloc"] + path = vendor/mimalloc + url = git@github.com:microsoft/mimalloc.git diff --git a/Makefile b/Makefile index 8c44a36e..61283568 100644 --- a/Makefile +++ b/Makefile @@ -95,13 +95,14 @@ test: .PHONY: install-submodule .PHONY: install-jsruntime install-jsruntime-dev install-libiconv .PHONY: _install-netsurf install-netsurf clean-netsurf test-netsurf install-netsurf-dev +.PHONY: install-mimalloc install-mimalloc-dev clean-mimalloc .PHONY: install-dev install ## Install and build dependencies for release -install: install-submodule install-jsruntime install-netsurf +install: install-submodule install-jsruntime install-netsurf install-mimalloc ## Install and build dependencies for dev -install-dev: install-submodule install-jsruntime-dev install-netsurf-dev +install-dev: install-submodule install-jsruntime-dev install-netsurf-dev install-mimalloc-dev install-netsurf-dev: _install-netsurf install-netsurf-dev: OPTCFLAGS := -O0 -g -DNDEBUG @@ -185,6 +186,24 @@ install-jsruntime: @cd vendor/jsruntime-lib && \ make install +install-mimalloc-dev: + @cd vendor/mimalloc && \ + mkdir -p out && \ + cd out && \ + cmake -DCMAKE_BUILD_TYPE=Debug .. && \ + make mimalloc-static && \ + mv libmimalloc-debug.a libmimalloc.a + +install-mimalloc: + @cd vendor/mimalloc && \ + mkdir -p out && \ + cd out && \ + cmake .. && \ + make mimalloc-static + +clean-mimalloc: + rm -fr vendor/mimalloc/lib/* + ## Init and update git submodule install-submodule: @git submodule init && \ diff --git a/vendor/mimalloc b/vendor/mimalloc new file mode 160000 index 00000000..8f7d1e9a --- /dev/null +++ b/vendor/mimalloc @@ -0,0 +1 @@ +Subproject commit 8f7d1e9a41bb0182166aac6a8d4d8b00f60ed032 From dad51a41795550c15f309a2221eef47f72a16654 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 9 Apr 2024 16:05:43 +0200 Subject: [PATCH 04/12] upgrade libwapcaplet deps --- vendor/netsurf/libwapcaplet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/netsurf/libwapcaplet b/vendor/netsurf/libwapcaplet index 80d955af..8b56983b 160000 --- a/vendor/netsurf/libwapcaplet +++ b/vendor/netsurf/libwapcaplet @@ -1 +1 @@ -Subproject commit 80d955afde761dece8a352cc5f85f2d4d624f4ab +Subproject commit 8b56983b4c2076d43c22ce1d391fab52c7764809 From 9363acf4ecfbbeff8f303bc1cbd7f87003306a44 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 9 Apr 2024 16:18:48 +0200 Subject: [PATCH 05/12] glue mimalloc with netsurf C libs --- build.zig | 4 ++++ src/mimalloc.zig | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ src/netsurf.zig | 20 ++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 src/mimalloc.zig diff --git a/build.zig b/build.zig index a414b2d6..e2e1a9c0 100644 --- a/build.zig +++ b/build.zig @@ -152,6 +152,10 @@ fn common( ) !void { try jsruntime_pkgs.add(step, options); linkNetSurf(step); + + // link mimalloc + step.addObjectFile(.{ .path = "vendor/mimalloc/out/libmimalloc.a" }); + step.addIncludePath(.{ .path = "vendor/mimalloc/include" }); } fn linkNetSurf(step: *std.build.LibExeObjStep) void { diff --git a/src/mimalloc.zig b/src/mimalloc.zig new file mode 100644 index 00000000..b9d563e4 --- /dev/null +++ b/src/mimalloc.zig @@ -0,0 +1,48 @@ +// This file makes the glue between mimalloc heap allocation and libdom memory +// management. +// We replace the libdom default usage of allocations with mimalloc heap +// allocation to be able to free all memory used at once, like an arena usage. + +const std = @import("std"); +const c = @cImport({ + @cInclude("mimalloc.h"); +}); + +const Error = error{ + HeapNotNull, + HeapNull, +}; + +var heap: ?*c.mi_heap_t = null; + +pub fn create() Error!void { + if (heap != null) return Error.HeapNotNull; + heap = c.mi_heap_new(); + if (heap == null) return Error.HeapNull; +} + +pub fn destroy() void { + if (heap == null) return; + c.mi_heap_destroy(heap.?); + heap = null; +} + +pub export fn m_alloc(size: usize) callconv(.C) ?*anyopaque { + if (heap == null) return null; + return c.mi_heap_malloc(heap.?, size); +} + +pub export fn re_alloc(ptr: ?*anyopaque, size: usize) callconv(.C) ?*anyopaque { + if (heap == null) return null; + return c.mi_heap_realloc(heap.?, ptr, size); +} + +pub export fn c_alloc(nmemb: usize, size: usize) callconv(.C) ?*anyopaque { + if (heap == null) return null; + return c.mi_heap_calloc(heap.?, nmemb, size); +} + +// NOOP, use destroy to clear all the memory allocated at once. +pub export fn f_ree(_: ?*anyopaque) callconv(.C) void { + return; +} diff --git a/src/netsurf.zig b/src/netsurf.zig index 619c37c7..1bc48e16 100644 --- a/src/netsurf.zig +++ b/src/netsurf.zig @@ -8,9 +8,29 @@ const c = @cImport({ @cInclude("events/event.h"); }); +const mimalloc = @import("mimalloc.zig"); + const Callback = @import("jsruntime").Callback; const EventToInterface = @import("events/event.zig").Event.toInterface; +// init initializes netsurf lib. +// init starts a mimalloc heap arena for the netsurf session. The caller must +// call deinit() to free the arena memory. +pub fn init() !void { + try mimalloc.create(); +} + +// deinit frees the mimalloc heap arena memory. +// It also clean dom namespaces and lwc strings. +pub fn deinit() void { + _ = c.dom_namespace_finalise(); + + // destroy all lwc strings. + c.lwc_deinit_strings(); + + mimalloc.destroy(); +} + // Vtable // ------ From 3834ebcfa4690be685e55fa09c1813d5583df652 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 9 Apr 2024 16:22:20 +0200 Subject: [PATCH 06/12] replace calloc with mimalloc --- src/browser/browser.zig | 6 ++ src/calloc.zig | 164 --------------------------------------- src/dom/namednodemap.zig | 4 +- src/main.zig | 3 + src/main_get.zig | 8 +- src/main_shell.zig | 6 +- src/run_tests.zig | 14 ++-- src/wpt/run.zig | 2 + 8 files changed, 22 insertions(+), 185 deletions(-) delete mode 100644 src/calloc.zig diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 6cd043c6..0508a391 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -134,6 +134,9 @@ pub const Page = struct { self.session.env.stop(); // TODO unload document: https://html.spec.whatwg.org/#unloading-documents + // clear netsurf memory arena. + parser.deinit(); + _ = self.arena.reset(.free_all); } @@ -211,6 +214,9 @@ pub const Page = struct { fn loadHTMLDoc(self: *Page, reader: anytype, charset: []const u8) !void { const alloc = self.arena.allocator(); + // start netsurf memory arena. + try parser.init(); + log.debug("parse html with charset {s}", .{charset}); const ccharset = try alloc.dupeZ(u8, charset); diff --git a/src/calloc.zig b/src/calloc.zig deleted file mode 100644 index fd2ceacb..00000000 --- a/src/calloc.zig +++ /dev/null @@ -1,164 +0,0 @@ -// MIT License -// Copyright 2024 Lightpanda -// Original copyright 2021 pfg and marler8997 -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -// associated documentation files (the "Software"), to deal in the Software without restriction, -// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, -// subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all copies or substantial -// portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT -// NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// --------------------------- - -// This file is a mix between: -// - malloc functions of ziglibc (https://github.com/marler8997/ziglibc/blob/main/src/cstd.zig) -// for the general logic -// - this gist https://gist.github.com/pfgithub/65c13d7dc889a4b2ba25131994be0d20 -// for the header "magic" validator -// + some refacto and comments to make the code more clear - -const std = @import("std"); - -// TODO: this uses a global variable -// it does not allow to set a context-based allocator -var alloc: std.mem.Allocator = undefined; - -pub fn setCAllocator(allocator: std.mem.Allocator) void { - alloc = allocator; -} - -// Alloc mechanism -// --------------- - -// C malloc does not know the type of the buffer allocated, -// instead it uses a metadata header at the begining of the buffer to store the allocated size. -// We copy this behavior by allocating in Zig the size requested + the size of the header. -// On this header we store not only the size allocated but also a "magic" validator -// to check if the C pointer as been allocated through those cutom malloc functions. - -// The total buffer looks like that: -// [Zig buf] = [header][C pointer] - -const al = @alignOf(std.c.max_align_t); - -const Header = struct { - comptime { - if (@alignOf(Header) > al) @compileError("oops"); - } - - const len = std.mem.alignForward(usize, al, @sizeOf(Header)); - - const MAGIC = 0xABCDEF; - const NOMAGIC = 0; - - magic: usize = MAGIC, - size: usize, -}; - -// Buffer manipulation functions - -// setHeader on a buffer allocated in Zig -inline fn setHeader(buf: anytype, size: usize) void { - // cast buffer to an header - const header = @as(*Header, @ptrCast(buf)); - // and set the relevant information on it (size and "magic" validator) - header.* = .{ .size = size }; -} - -// getHeader from a C pointer -fn getHeader(ptr: [*]u8) *Header { - // use arithmetic to get (ie. backward) the buffer pointer from the C pointer - const buf = ptr - Header.len; - // convert many-item pointer to single pointer and cast to an header - // return @ptrFromInt(@intFromPtr(buf)); - // and cast it to an header pointer - return @ptrCast(@as([*]align(@alignOf(*Header)) u8, @alignCast(buf))); -} - -// getBuf from an header -fn getBuf(header: *Header) []align(al) u8 { - // cast header pointer to a many-item buffer pointer - const buf_ptr = @as([*]u8, @ptrCast(header)); - // return the buffer with corresponding length - const buf = buf_ptr[0..header.size]; - return @alignCast(buf); -} - -inline fn cPtr(buf: [*]align(al) u8) [*]align(al) u8 { - // use arithmetic to get (ie. forward) the C pointer from the buffer pointer - return buf + Header.len; -} - -// Custom malloc functions - -pub export fn m_alloc(size: usize) callconv(.C) ?[*]align(al) u8 { - std.debug.assert(size > 0); // TODO: what should we do in this case? - const buf_len = Header.len + size; - const buf = alloc.alignedAlloc(u8, al, buf_len) catch |err| switch (err) { - error.OutOfMemory => return null, - }; - setHeader(buf, buf_len); - return cPtr(buf.ptr); -} - -pub export fn re_alloc(ptr: ?[*]align(al) u8, size: usize) callconv(.C) ?[*]align(al) u8 { - if (ptr == null) return m_alloc(size); - const header = getHeader(ptr.?); - const buf = getBuf(header); - if (size == 0) { - alloc.free(buf); - return null; - } - - const buf_len = Header.len + size; - if (alloc.rawResize(buf, std.math.log2(al), buf_len, @returnAddress())) { - setHeader(buf.ptr, buf_len); - return ptr; - } - - const new_buf = alloc.reallocAdvanced( - buf, - buf_len, - @returnAddress(), - ) catch |e| switch (e) { - error.OutOfMemory => return null, - }; - setHeader(new_buf.ptr, buf_len); - return cPtr(new_buf.ptr); -} - -export fn c_alloc(nmemb: usize, size: usize) callconv(.C) ?[*]align(al) u8 { - const total = std.math.mul(usize, nmemb, size) catch { - // TODO: set errno - // errno = c.ENOMEM; - return null; - }; - const ptr = m_alloc(total) orelse return null; - @memset(ptr[0..total], 0); - return ptr; -} - -pub export fn f_ree(ptr: ?[*]align(al) u8) callconv(.C) void { - if (ptr == null) return; - - // check header - const header = getHeader(ptr.?); - if (header.magic != Header.MAGIC) { - // either doble-free or allocated outside those custom mallocs - // TODO: why? - return; - } - header.magic = Header.NOMAGIC; // prevent double free - - const buf = getBuf(header); - alloc.free(buf); -} diff --git a/src/dom/namednodemap.zig b/src/dom/namednodemap.zig index 3e1793f1..cbca8c7d 100644 --- a/src/dom/namednodemap.zig +++ b/src/dom/namednodemap.zig @@ -73,9 +73,7 @@ pub fn testExecFn( .{ .src = "a.item(1)", .ex = "null" }, .{ .src = "a.getNamedItem('id')", .ex = "[object Attr]" }, .{ .src = "a.getNamedItem('foo')", .ex = "null" }, - // TODO: with setCAllocator this test fails with a segfault - // see https://github.com/lightpanda-io/browsercore/issues/197 - // .{ .src = "a.setNamedItem(a.getNamedItem('id'))", .ex = "[object Attr]" }, + .{ .src = "a.setNamedItem(a.getNamedItem('id'))", .ex = "[object Attr]" }, }; try checkCases(js_env, &setItem); } diff --git a/src/main.zig b/src/main.zig index 1dc34dc6..2d6aa4bc 100644 --- a/src/main.zig +++ b/src/main.zig @@ -56,6 +56,9 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); + try parser.init(); + defer parser.deinit(); + // document const file = try std.fs.cwd().openFile("test.html", .{}); defer file.close(); diff --git a/src/main_get.zig b/src/main_get.zig index 15c4f90e..de6191c6 100644 --- a/src/main_get.zig +++ b/src/main_get.zig @@ -2,7 +2,6 @@ const std = @import("std"); const Browser = @import("browser/browser.zig").Browser; const jsruntime = @import("jsruntime"); -const setCAllocator = @import("calloc.zig").setCAllocator; const apiweb = @import("apiweb.zig"); pub const Types = jsruntime.reflect(apiweb.Interfaces); @@ -30,10 +29,6 @@ pub fn main() !void { } const allocator = gpa.allocator(); - var c_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer c_arena.deinit(); - setCAllocator(c_arena.allocator()); - var args = try std.process.argsWithAllocator(allocator); defer args.deinit(); @@ -70,9 +65,10 @@ pub fn main() !void { defer browser.deinit(); var page = try browser.currentSession().createPage(); - defer page.end(); + defer page.deinit(); try page.navigate(url); + defer page.end(); if (dump) { try page.dump(std.io.getStdOut()); diff --git a/src/main_shell.zig b/src/main_shell.zig index f6d4941c..43271b3d 100644 --- a/src/main_shell.zig +++ b/src/main_shell.zig @@ -1,7 +1,6 @@ const std = @import("std"); const jsruntime = @import("jsruntime"); -const setCAllocator = @import("calloc.zig").setCAllocator; const parser = @import("netsurf.zig"); const apiweb = @import("apiweb.zig"); @@ -38,9 +37,8 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(gpa.allocator()); defer arena.deinit(); - var c_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer c_arena.deinit(); - setCAllocator(c_arena.allocator()); + try parser.init(); + defer parser.deinit(); // document const file = try std.fs.cwd().openFile("test.html", .{}); diff --git a/src/run_tests.zig b/src/run_tests.zig index 8220336c..14a235b7 100644 --- a/src/run_tests.zig +++ b/src/run_tests.zig @@ -5,7 +5,6 @@ const jsruntime = @import("jsruntime"); const generate = @import("generate.zig"); const pretty = @import("pretty"); -const setCAllocator = @import("calloc.zig").setCAllocator; const parser = @import("netsurf.zig"); const apiweb = @import("apiweb.zig"); const Window = @import("html/window.zig").Window; @@ -39,6 +38,9 @@ fn testExecFn( js_env: *jsruntime.Env, comptime execFn: jsruntime.ContextExecFn, ) anyerror!void { + try parser.init(); + defer parser.deinit(); + // start JS env try js_env.start(alloc); defer js_env.stop(); @@ -88,7 +90,6 @@ fn testsAllExecFn( inline for (testFns) |testFn| { try testExecFn(alloc, js_env, testFn); - _ = c_arena.reset(.retain_capacity); } } @@ -117,8 +118,6 @@ const Run = enum { unit, }; -var c_arena: std.heap.ArenaAllocator = undefined; - pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); @@ -152,10 +151,6 @@ pub fn main() !void { } } - c_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer c_arena.deinit(); - setCAllocator(c_arena.allocator()); - // run js tests if (run == .all or run == .browser) try run_js(out); @@ -163,6 +158,9 @@ pub fn main() !void { if (run == .all or run == .unit) { std.debug.print("\n", .{}); for (builtin.test_functions) |test_fn| { + try parser.init(); + defer parser.deinit(); + try test_fn.func(); std.debug.print("{s}\tOK\n", .{test_fn.name}); } diff --git a/src/wpt/run.zig b/src/wpt/run.zig index 686363c0..467436a5 100644 --- a/src/wpt/run.zig +++ b/src/wpt/run.zig @@ -17,6 +17,8 @@ const Types = @import("../main_wpt.zig").Types; // It loads first the js libs files. pub fn run(arena: *std.heap.ArenaAllocator, comptime dir: []const u8, f: []const u8, loader: *FileLoader) !jsruntime.JSResult { const alloc = arena.allocator(); + try parser.init(); + defer parser.deinit(); // document const file = try std.fs.cwd().openFile(f, .{}); From 53a53262483517fca0b23cdf7f0eb2c2a56d9995 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Wed, 10 Apr 2024 09:18:42 +0200 Subject: [PATCH 07/12] mimalloc: avoid mimalloc override By default mimalloc is built to override default allocation functions. So it is used also by v8. This change avoid the mimalloc override to keep the native stdlib functions. --- Makefile | 24 +++++++++++++----------- build.zig | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 61283568..42ee34f7 100644 --- a/Makefile +++ b/Makefile @@ -186,23 +186,25 @@ install-jsruntime: @cd vendor/jsruntime-lib && \ make install -install-mimalloc-dev: +.PHONY: _build_mimalloc +_build_mimalloc: @cd vendor/mimalloc && \ - mkdir -p out && \ + mkdir -p out/include && \ + cp include/mimalloc.h out/include/ && \ cd out && \ - cmake -DCMAKE_BUILD_TYPE=Debug .. && \ - make mimalloc-static && \ + cmake -DMI_BUILD_SHARED=OFF -DMI_BUILD_OBJECT=OFF -DMI_BUILD_TESTS=OFF -DMI_OVERRIDE=OFF $(OPTS) .. && \ + make + +install-mimalloc-dev: _build_mimalloc +install-mimalloc-dev: OPTS=-DCMAKE_BUILD_TYPE=Debug +install-mimalloc-dev: + @cd vendor/mimalloc/out && \ mv libmimalloc-debug.a libmimalloc.a -install-mimalloc: - @cd vendor/mimalloc && \ - mkdir -p out && \ - cd out && \ - cmake .. && \ - make mimalloc-static +install-mimalloc: _build_mimalloc clean-mimalloc: - rm -fr vendor/mimalloc/lib/* + @rm -fr vendor/mimalloc/lib/* ## Init and update git submodule install-submodule: diff --git a/build.zig b/build.zig index e2e1a9c0..c5b8be86 100644 --- a/build.zig +++ b/build.zig @@ -155,7 +155,7 @@ fn common( // link mimalloc step.addObjectFile(.{ .path = "vendor/mimalloc/out/libmimalloc.a" }); - step.addIncludePath(.{ .path = "vendor/mimalloc/include" }); + step.addIncludePath(.{ .path = "vendor/mimalloc/out/include" }); } fn linkNetSurf(step: *std.build.LibExeObjStep) void { From 69b5a3db15a05efcf701c9bf2e8afb830492194d Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Wed, 10 Apr 2024 09:27:24 +0200 Subject: [PATCH 08/12] readme: add mimalloc info --- README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a78620ae..f4054973 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,9 @@ Browsercore is written with [Zig](https://ziglang.org/) `0.12`. You have to install it with the right version in order to build the project. Browsercore also depends on -[js-runtimelib](https://github.com/francisbouvier/jsruntime-lib/) and -[Netsurf libs](https://www.netsurf-browser.org/) libs. +[js-runtimelib](https://github.com/francisbouvier/jsruntime-lib/), +[Netsurf libs](https://www.netsurf-browser.org/) and +[Mimalloc](https://microsoft.github.io/mimalloc) libs. To be able to build the v8 engine for js-runtimelib, you have to install some libs: @@ -36,11 +37,26 @@ make install-submodule ### Build Netsurf -The command `make install-netsurf` will build netsurf libs used by browsercore. +The command `make install-netsurf` will build Netsurf libs used by browsercore. ``` make install-netsurf ``` +For dev env, use `make install-netsurf-dev`. + +### Build Mimalloc + +The command `make install-mimalloc` will build Mimalloc lib used by browsercore. +``` +make install-mimalloc +``` + +For dev env, use `make install-mimalloc-dev`. + +Note, when Mimalloc is built in dev mode, you can dump memory stats with the +env var `MIMALLOC_SHOW_STATS=1`. See +https://microsoft.github.io/mimalloc/environment.html + ### Build jsruntime-lib The command `make install-jsruntime-dev` uses jsruntime-lib's `zig-v8` dependency to build v8 engine lib. From 545bcc403a6bee0e8fd8fc955fb839e3026c377a Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 16 Apr 2024 16:27:37 +0200 Subject: [PATCH 09/12] ci: rebuild mimalloc if it has changed --- .github/actions/install/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 2b8f76dc..0e285631 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -19,6 +19,10 @@ runs: run: | ln -s /usr/local/lib/libiconv vendor/libiconv + - name: build mimalloc + shell: bash + run: make install-mimalloc + - name: build netsurf shell: bash run: make install-netsurf From a3e91debea2f38a5576cd2db7bb9ada55e5b9b60 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 16 Apr 2024 16:34:08 +0200 Subject: [PATCH 10/12] deps: upgrade netsurf deps --- vendor/netsurf/libdom | 2 +- vendor/netsurf/libhubbub | 2 +- vendor/netsurf/libparserutils | 2 +- vendor/netsurf/libwapcaplet | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vendor/netsurf/libdom b/vendor/netsurf/libdom index 6c87ab81..293790bb 160000 --- a/vendor/netsurf/libdom +++ b/vendor/netsurf/libdom @@ -1 +1 @@ -Subproject commit 6c87ab81ce6c1a0020267dff27b32da83f73e796 +Subproject commit 293790bb9336961a5c7b0e38fd9f254146eee8d0 diff --git a/vendor/netsurf/libhubbub b/vendor/netsurf/libhubbub index 79d33036..6f102212 160000 --- a/vendor/netsurf/libhubbub +++ b/vendor/netsurf/libhubbub @@ -1 +1 @@ -Subproject commit 79d3303613a5dacb90e1e42129ab0e7ab2068b79 +Subproject commit 6f102212c85f48a3c61c916ea46a74a37a2bfc9b diff --git a/vendor/netsurf/libparserutils b/vendor/netsurf/libparserutils index fb7b7eaf..094dc22e 160000 --- a/vendor/netsurf/libparserutils +++ b/vendor/netsurf/libparserutils @@ -1 +1 @@ -Subproject commit fb7b7eaf3d37ad8bcd795bbaaa3d0d93fc8cc5da +Subproject commit 094dc22e2b3c21e8d12f2275fd7bf09bc4da3f3e diff --git a/vendor/netsurf/libwapcaplet b/vendor/netsurf/libwapcaplet index 8b56983b..74f1e011 160000 --- a/vendor/netsurf/libwapcaplet +++ b/vendor/netsurf/libwapcaplet @@ -1 +1 @@ -Subproject commit 8b56983b4c2076d43c22ce1d391fab52c7764809 +Subproject commit 74f1e0117310b5392da484a71346cf09f78e8216 From 304a28a79d4cf1997d40800472029462ba08a222 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Thu, 18 Apr 2024 15:39:13 +0200 Subject: [PATCH 11/12] mimalloc: add strdup and strndup overrride --- src/mimalloc.zig | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/mimalloc.zig b/src/mimalloc.zig index b9d563e4..58f98780 100644 --- a/src/mimalloc.zig +++ b/src/mimalloc.zig @@ -42,6 +42,16 @@ pub export fn c_alloc(nmemb: usize, size: usize) callconv(.C) ?*anyopaque { return c.mi_heap_calloc(heap.?, nmemb, size); } +pub export fn str_dup(s: [*c]const u8) callconv(.C) [*c]u8 { + if (heap == null) return null; + return c.mi_heap_strdup(heap.?, s); +} + +pub export fn strn_dup(s: [*c]const u8, size: usize) callconv(.C) [*c]u8 { + if (heap == null) return null; + return c.mi_heap_strndup(heap.?, s, size); +} + // NOOP, use destroy to clear all the memory allocated at once. pub export fn f_ree(_: ?*anyopaque) callconv(.C) void { return; From 8e96ee337dce27f4c5123ac23216f74dc4c8d381 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Fri, 19 Apr 2024 11:55:55 +0200 Subject: [PATCH 12/12] wpt: skip tests/wpt/dom/events/remove-all-listeners.html --- tests/wpt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wpt b/tests/wpt index 46948926..735b2182 160000 --- a/tests/wpt +++ b/tests/wpt @@ -1 +1 @@ -Subproject commit 4694892636c99fac364a203fd3b335dd7199247c +Subproject commit 735b21823e1d3e59f4ff1946612293be196dfc36