incremental(windows) add TransmitFile and some fixes for fetch, h2 an…

…d http (#8089) * fix win header decoding zig issue * fix libuv file read lifecycle * upload working without sendfile (yet) * undo * oops uncoment expect * add TransmitFile aka sendfile on windows * cleanup * [autofix.ci] apply automated fixes --------- Co-authored-by: cirospaciari <[email protected]> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Jarred Sumner <[email protected]>
oven-sh · Jan 12, 2024 · 4611b84 · Jarred-Sumner · Jan 23, 2024 · cirospaciari
1 parent 922ff08
commit 4611b84
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 45 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1311,6 +1311,7 @@ else()
         ucrt
         userenv
         dbghelp
+        wsock32 # ws2_32 required by TransmitFile aka sendfile on windows
     )
 endif()
 

diff --git a/src/bun.js/api/bun/h2_frame_parser.zig b/src/bun.js/api/bun/h2_frame_parser.zig
@@ -669,19 +669,17 @@ pub const H2FrameParser = struct {
         var xhdr: lshpack.lsxpack_header = .{};
 
         lshpack.lsxpack_header_prepare_decode(&xhdr, header_buffer.ptr, 0, header_buffer.len);
-        const start = @intFromPtr(src_buffer.ptr);
-        var src = src_buffer.ptr;
-        if (lshpack.lshpack_dec_decode(&this.decoder, &src, @ptrFromInt(start + src_buffer.len), &xhdr) != 0) {
-            return error.UnableToDecode;
-        }
+        const next = try lshpack.lshpack_decode(&this.decoder, src_buffer.ptr, src_buffer.len, &xhdr);
+
         const name = lshpack.lsxpack_header_get_name(&xhdr);
         if (name.len == 0) {
             return error.EmptyHeaderName;
         }
+
         return .{
             .name = name,
             .value = lshpack.lsxpack_header_get_value(&xhdr),
-            .next = @intFromPtr(src) - start,
+            .next = next,
         };
     }
 

diff --git a/src/bun.js/api/bun/lshpack.translated.zig b/src/bun.js/api/bun/lshpack.translated.zig
@@ -220,38 +220,38 @@ pub fn lsxpack_header_mark_val_changed(hdr: ?*lsxpack_header_t) callconv(.C) voi
 }
 pub const struct_lshpack_enc_table_entry = opaque {};
 pub const struct_lshpack_enc_head = extern struct {
-    stqh_first: ?*struct_lshpack_enc_table_entry,
-    stqh_last: [*c]?*struct_lshpack_enc_table_entry,
+    stqh_first: ?*struct_lshpack_enc_table_entry = @import("std").mem.zeroes(?*struct_lshpack_enc_table_entry),
+    stqh_last: [*c]?*struct_lshpack_enc_table_entry = @import("std").mem.zeroes([*c]?*struct_lshpack_enc_table_entry),
 };
 pub const struct_lshpack_double_enc_head = opaque {};
 pub const LSHPACK_ENC_USE_HIST: c_int = 1;
 const enum_unnamed_1 = c_uint;
 pub const struct_lshpack_enc = extern struct {
-    hpe_cur_capacity: c_uint,
-    hpe_max_capacity: c_uint,
-    hpe_next_id: c_uint,
-    hpe_nelem: c_uint,
-    hpe_nbits: c_uint,
-    hpe_all_entries: struct_lshpack_enc_head,
-    hpe_buckets: ?*struct_lshpack_double_enc_head,
-    hpe_hist_buf: [*c]u32,
-    hpe_hist_size: c_uint,
-    hpe_hist_idx: c_uint,
-    hpe_hist_wrapped: c_int,
-    hpe_flags: enum_unnamed_1,
+    hpe_cur_capacity: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_max_capacity: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_next_id: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_nelem: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_nbits: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_all_entries: struct_lshpack_enc_head = @import("std").mem.zeroes(struct_lshpack_enc_head),
+    hpe_buckets: ?*struct_lshpack_double_enc_head = @import("std").mem.zeroes(?*struct_lshpack_double_enc_head),
+    hpe_hist_buf: [*c]u32 = @import("std").mem.zeroes([*c]u32),
+    hpe_hist_size: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_hist_idx: c_uint = @import("std").mem.zeroes(c_uint),
+    hpe_hist_wrapped: c_int = @import("std").mem.zeroes(c_int),
+    hpe_flags: enum_unnamed_1 = @import("std").mem.zeroes(enum_unnamed_1),
 };
 pub const struct_lshpack_arr = extern struct {
-    nalloc: c_uint,
-    nelem: c_uint,
-    off: c_uint,
-    els: [*c]usize,
+    nalloc: c_uint = @import("std").mem.zeroes(c_uint),
+    nelem: c_uint = @import("std").mem.zeroes(c_uint),
+    off: c_uint = @import("std").mem.zeroes(c_uint),
+    els: [*c]usize = @import("std").mem.zeroes([*c]usize),
 };
 pub const struct_lshpack_dec = extern struct {
-    hpd_dyn_table: struct_lshpack_arr,
-    hpd_max_capacity: c_uint,
-    hpd_cur_max_capacity: c_uint,
-    hpd_cur_capacity: c_uint,
-    hpd_state: c_uint,
+    hpd_dyn_table: struct_lshpack_arr = @import("std").mem.zeroes(struct_lshpack_arr),
+    hpd_max_capacity: c_uint = @import("std").mem.zeroes(c_uint),
+    hpd_cur_max_capacity: c_uint = @import("std").mem.zeroes(c_uint),
+    hpd_cur_capacity: c_uint = @import("std").mem.zeroes(c_uint),
+    hpd_state: c_uint = @import("std").mem.zeroes(c_uint),
 };
 pub const LSHPACK_HDR_UNKNOWN: c_int = 0;
 pub const LSHPACK_HDR_AUTHORITY: c_int = 1;
@@ -325,10 +325,17 @@ pub extern fn lshpack_enc_use_hist([*c]struct_lshpack_enc, on: c_int) c_int;
 pub extern fn lshpack_enc_hist_used([*c]const struct_lshpack_enc) c_int;
 pub extern fn lshpack_dec_init([*c]struct_lshpack_dec) void;
 pub extern fn lshpack_dec_cleanup([*c]struct_lshpack_dec) void;
-pub extern fn lshpack_dec_decode(dec: [*c]struct_lshpack_dec, src: *[*]const u8, src_end: [*c]const u8, output: ?*struct_lsxpack_header) c_int;
+pub extern fn lshpack_dec_decode(dec: [*c]struct_lshpack_dec, src: *[*c]const u8, src_end: [*c]const u8, output: ?*struct_lsxpack_header) c_int;
 pub extern fn lshpack_dec_set_max_capacity([*c]struct_lshpack_dec, c_uint) void;
 pub extern fn lshpack_enc_get_stx_tab_id(?*struct_lsxpack_header) c_uint;
-
+pub fn lshpack_decode(dec: [*c]struct_lshpack_dec, src: [*]const u8, src_len: usize, output: ?*struct_lsxpack_header) !usize {
+    var s: [*c]const u8 = src;
+    const rc: c_int = lshpack_dec_decode(dec, &s, s + src_len, output);
+    if (rc != 0) {
+        return error.UnableToDecode;
+    }
+    return @intFromPtr(s) - @intFromPtr(src);
+}
 pub const __INT64_C = @import("std").zig.c_translation.Macros.L_SUFFIX;
 pub const __UINT64_C = @import("std").zig.c_translation.Macros.UL_SUFFIX;
 pub const INT8_MIN = -@as(c_int, 128);

diff --git a/src/bun.js/api/server.zig b/src/bun.js/api/server.zig
@@ -80,7 +80,7 @@ const BoringSSL = @import("root").bun.BoringSSL;
 const Arena = @import("../../mimalloc_arena.zig").Arena;
 const SendfileContext = struct {
     fd: bun.FileDescriptor,
-    socket_fd: i32 = 0,
+    socket_fd: bun.FileDescriptor = bun.invalid_fd,
     remain: Blob.SizeType = 0,
     offset: Blob.SizeType = 0,
     has_listener: bool = false,
@@ -1277,8 +1277,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
         defer_deinit_until_callback_completes: ?*bool = null,
 
         // TODO: support builtin compression
-        // TODO: Use TransmitFile on Windows
-        const can_sendfile = !ssl_enabled and !bun.Environment.isWindows;
+        const can_sendfile = !ssl_enabled;
 
         pub inline fn isAsync(this: *const RequestContext) bool {
             return this.defer_deinit_until_callback_completes == null;
@@ -1894,6 +1893,14 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
                     this.cleanupAndFinalizeAfterSendfile();
                     return errcode != .SUCCESS;
                 }
+            } else if(Environment.isWindows) {
+                const win = std.os.windows;
+                const uv = bun.windows.libuv;
+                const socket = bun.socketcast(this.sendfile.socket_fd);
+                const file_handle = uv.uv_get_osfhandle(bun.uvfdcast(this.sendfile.fd));
+                this.sendfile.offset += this.sendfile.remain;
+                this.sendfile.remain = 0;
+                return win.ws2_32.TransmitFile(socket, file_handle, 0, 0, null, null, 0) == 1;
             } else {
                 var sbytes: std.os.off_t = adjusted_count;
                 const signed_offset = @as(i64, @bitCast(@as(u64, this.sendfile.offset)));
@@ -2066,7 +2073,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
                 .remain = this.blob.Blob.offset + original_size,
                 .offset = this.blob.Blob.offset,
                 .auto_close = auto_close,
-                .socket_fd = if (!this.flags.aborted) resp.getNativeHandle() else -999,
+                .socket_fd = if (!this.flags.aborted) resp.getNativeHandle() else bun.invalid_fd,
             };
 
             // if we are sending only part of a file, include the content-range header
@@ -2158,7 +2165,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
                     .remain = @as(Blob.SizeType, @truncate(result.result.buf.len)),
                     .offset = if (this.blob == .Blob) this.blob.Blob.offset else 0,
                     .auto_close = false,
-                    .socket_fd = -999,
+                    .socket_fd = bun.invalid_fd,
                 };
 
                 this.response_buf_owned = .{ .items = result.result.buf, .capacity = result.result.buf.len };

diff --git a/src/bun.js/webcore/blob.zig b/src/bun.js/webcore/blob.zig
@@ -2330,16 +2330,17 @@ pub const Blob = struct {
 
             req: libuv.fs_t = libuv.fs_t.uninitialized,
 
-            pub fn start(loop: *libuv.Loop, store: *Store, off: SizeType, max_len: SizeType, comptime Handler: type, handler: *Handler) void {
+            pub fn start(loop: *libuv.Loop, store: *Store, off: SizeType, max_len: SizeType, comptime Handler: type, handler: *anyopaque) void {
                 var this = bun.new(ReadFileUV, .{
                     .loop = loop,
                     .file_store = store.data.file,
                     .store = store,
                     .offset = off,
                     .max_length = max_len,
-                    .on_complete_data = @ptrCast(handler),
+                    .on_complete_data = handler,
                     .on_complete_fn = @ptrCast(&Handler.run),
                 });
+                store.ref();
                 this.getFd(onFileOpen);
             }
 
@@ -4361,7 +4362,8 @@ pub const Blob = struct {
 
     pub fn doReadFileInternal(this: *Blob, comptime Handler: type, ctx: Handler, comptime Function: anytype, global: *JSGlobalObject) void {
         if (Environment.isWindows) {
-            @panic("todo");
+            const ReadFileHandler = NewInternalReadFileHandler(Handler, Function);
+            return Store.ReadFileUV.start(libuv.Loop.get(), this.store.?, this.offset, this.size, ReadFileHandler, ctx);
         }
         const file_read = Store.ReadFile.createWithCtx(
             bun.default_allocator,

diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig
@@ -2220,7 +2220,7 @@ pub const Fetch = struct {
             prepare_body: {
                 const opened_fd_res: JSC.Node.Maybe(bun.FileDescriptor) = switch (body.Blob.store.?.data.file.pathlike) {
                     .fd => |fd| bun.sys.dup(fd),
-                    .path => |path| bun.sys.open(path.sliceZ(&globalThis.bunVM().nodeFS().sync_error_buf), std.os.O.RDONLY | std.os.O.NOCTTY, 0),
+                    .path => |path| bun.sys.open(path.sliceZ(&globalThis.bunVM().nodeFS().sync_error_buf), if(Environment.isWindows) std.os.O.RDONLY else std.os.O.RDONLY | std.os.O.NOCTTY, 0),
                 };
 
                 const opened_fd = switch (opened_fd_res) {

diff --git a/src/deps/uws.zig b/src/deps/uws.zig
@@ -209,12 +209,17 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type {
             return @as(*NativeSocketHandleType(is_ssl), @ptrCast(us_socket_get_native_handle(comptime ssl_int, this.socket).?));
         }
 
-        pub inline fn fd(this: ThisSocket) i32 {
+        pub inline fn fd(this: ThisSocket) bun.FileDescriptor {
             if (comptime is_ssl) {
                 @compileError("SSL sockets do not have a file descriptor accessible this way");
             }
 
-            return @as(i32, @intCast(@intFromPtr(us_socket_get_native_handle(0, this.socket))));
+            if (comptime Environment.isWindows) {
+                // on windows uSockets exposes SOCKET
+                return bun.toFD(@as(bun.FDImpl.System, @ptrCast(us_socket_get_native_handle(0, this.socket))));
+            }
+
+            return bun.toFD(@as(i32, @intCast(@intFromPtr(us_socket_get_native_handle(0, this.socket)))));
         }
 
         pub fn markNeedsMoreForSendfile(this: ThisSocket) void {
@@ -1948,8 +1953,13 @@ pub fn NewApp(comptime ssl: bool) type {
                 return uws_res_has_responded(ssl_flag, res.downcast());
             }
 
-            pub fn getNativeHandle(res: *Response) i32 {
-                return @as(i32, @intCast(@intFromPtr(uws_res_get_native_handle(ssl_flag, res.downcast()))));
+            pub fn getNativeHandle(res: *Response) bun.FileDescriptor {
+                if (comptime Environment.isWindows) {
+                    // on windows uSockets exposes SOCKET
+                    return bun.toFD(@as(bun.FDImpl.System, @ptrCast(uws_res_get_native_handle(ssl_flag, res.downcast()))));
+                }
+
+                return bun.toFD(@as(i32, @intCast(@intFromPtr(uws_res_get_native_handle(ssl_flag, res.downcast())))));
             }
             pub fn getRemoteAddress(res: *Response) ?[]const u8 {
                 var buf: [*]const u8 = undefined;

diff --git a/src/http.zig b/src/http.zig
@@ -115,6 +115,16 @@ pub const HTTPRequestBody = union(enum) {
     }
 };
 
+pub fn canUseBrotli() bool {
+    if (Environment.isMac) {
+        if (bun.CompressionFramework.isAvailable()) {
+            return true;
+        }
+    }
+
+    return bun.brotli.hasBrotli();
+}
+
 pub const Sendfile = struct {
     fd: bun.FileDescriptor,
     remain: usize = 0,
@@ -152,6 +162,18 @@ pub const Sendfile = struct {
 
                 return .{ .err = bun.errnoToZigErr(errcode) };
             }
+        } else if (Environment.isWindows) {
+            const win = std.os.windows;
+            const uv = bun.windows.libuv;
+            const wsocket = bun.socketcast(socket.fd());
+            const file_handle = uv.uv_get_osfhandle(bun.uvfdcast(this.fd));
+            if (win.ws2_32.TransmitFile(wsocket, file_handle, 0, 0, null, null, 0) == 1) {
+                return .{ .done = {} };
+            }
+            this.offset += this.remain;
+            this.remain = 0;
+            const errorno = win.ws2_32.WSAGetLastError();
+            return .{ .err = bun.errnoToZigErr(errorno) };
         } else if (Environment.isPosix) {
             var sbytes: std.os.off_t = adjusted_count;
             const signed_offset = @as(i64, @bitCast(@as(u64, this.offset)));

diff --git a/src/sys.zig b/src/sys.zig
@@ -653,6 +653,9 @@ pub fn openA(file_path: []const u8, flags: bun.Mode, perm: bun.Mode) Maybe(bun.F
 }
 
 pub fn open(file_path: [:0]const u8, flags: bun.Mode, perm: bun.Mode) Maybe(bun.FileDescriptor) {
+    if (comptime Environment.isWindows) {
+        return sys_uv.open(file_path, flags, perm);
+    }
     // this is what open() does anyway.
     return openat(bun.toFD((std.fs.cwd().fd)), file_path, flags, perm);
 }

diff --git a/src/sys_uv.zig b/src/sys_uv.zig
@@ -27,7 +27,8 @@ comptime {
 
 pub const log = bun.sys.syslog;
 pub const Error = bun.sys.Error;
-pub const open = bun.sys.open;
+
+// libuv dont suppport openat (https://github.com/libuv/libuv/issues/4167)
 pub const openat = bun.sys.openat;
 pub const getFdPath = bun.sys.getFdPath;
 pub const setFileOffset = bun.sys.setFileOffset;
@@ -36,6 +37,17 @@ pub const mkdirOSPath = bun.sys.mkdirOSPath;
 
 // Note: `req = undefined; req.deinit()` has a saftey-check in a debug build
 
+pub fn open(file_path: [:0]const u8, flags: bun.Mode, perm: bun.Mode) Maybe(bun.FileDescriptor) {
+    var req: uv.fs_t = uv.fs_t.uninitialized;
+    defer req.deinit();
+    const rc = uv.uv_fs_open(uv.Loop.get(), &req, file_path.ptr, flags, perm, null);
+    log("uv open({s}, {d}, {d}) = {d}", .{ file_path, flags, perm, rc.value });
+    return if (rc.errno()) |errno|
+        .{ .err = .{ .errno = errno, .syscall = .open } }
+    else
+        .{ .result = bun.toFD(@as(i32, @intCast(req.result.value))) };
+}
+
 pub fn mkdir(file_path: [:0]const u8, flags: bun.Mode) Maybe(void) {
     var req: uv.fs_t = uv.fs_t.uninitialized;
     defer req.deinit();
-Original file line number
+Diff line change
@@ Expand Up / @@ -1311,6 +1311,7 @@ else() @@
             ucrt
             userenv
             dbghelp
+            wsock32 # ws2_32 required by TransmitFile aka sendfile on windows
         )
     endif()
@@ Expand Down @@