Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework Zig SDK #2

Open
wants to merge 26 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
6d07f12
chore: zig fmt fixes in Zig 0.12
sea-grass Feb 2, 2024
6cf58b5
feat: add Zig wws module (Fixes #218)
sea-grass Feb 2, 2024
ccb6c79
chore: simplify zig wws request parsing
sea-grass Feb 3, 2024
77a19f2
chore: simplify Zig SDK reading request/writing response
sea-grass Feb 3, 2024
9d1c6f9
chore: move examples/zig-module to examples/zig-examples
sea-grass Feb 3, 2024
354a053
feat: add Var to Zig SDK
sea-grass Feb 6, 2024
7ab30fe
chore: Expose `addWorker` from Zig wws SDK
sea-grass Feb 9, 2024
d3ad559
chore: Refactor zig-examples build
sea-grass Feb 9, 2024
ba2bf90
chore: move Zig examples into `zig-examples`
sea-grass Feb 17, 2024
24639a5
feat: Update Zig kit for mount config options
sea-grass Feb 17, 2024
c895088
chore: Zig examples use the Zig wws kit
sea-grass Feb 17, 2024
b2fc796
chore: Update build.zig to include image for folder mount example
sea-grass Feb 17, 2024
640cd02
feat: Add Zig dynamic routing example using `zig-router`
sea-grass Feb 17, 2024
86e397b
feat: Add Zig no-alloc-kv example that doesn't make heap allocations
sea-grass Feb 18, 2024
6d47f9b
feat: Add Zig example that only makes dynamic allocations if necessary
sea-grass Feb 18, 2024
80915d1
chore: Update Zig examples for latest Zig 0.12.0-dev.2809+f3bd17772
sea-grass Feb 18, 2024
cdbe08b
update: bump zig-router and uncomment some Zig examples
sea-grass Feb 20, 2024
1a7d3e5
fix: Add missing features to Zig examples
sea-grass Feb 23, 2024
dca792b
chore: Simplify Zig examples
sea-grass Feb 25, 2024
a1a4906
chore: Update Zig docs
sea-grass Feb 25, 2024
9751886
fix: Zig example on `0.12.0-dev.3381+7057bffc1`
sea-grass Mar 20, 2024
aae9225
chore: remove Zig worker SDK in favour of Zig wws SDK
sea-grass Mar 22, 2024
5b7bcc9
chore: Update to Zig 0.13.0-dev.249+ed75f6256
sea-grass May 23, 2024
b30cc62
Add .zig-cache to .gitignore
sea-grass Jan 28, 2025
47a0c5d
Remove zig-router dependency to ease maintenance burden of zig examples
sea-grass Jan 28, 2025
c346462
Update Zig docs
sea-grass Jan 28, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: add Zig wws module (Fixes vmware-labs#218)
This commit introduces `kits/zig/wws`, with the intention of replacing `kits/zig/worker`.
The design of the new SDK improves upon the previous one by exposing utilities for parsing wws requests and serializing wws responses, to give the user more control over memory while still keeping the SDK easy to use.
The name was changed to make it more ergonomic within the Zig ecosystem (the developer will pull in a `wws` dependency to interface with wws, rather than a `worker` dependency).
sea-grass committed Jan 28, 2025
commit 6cf58b58642913b8d147159cd6aebf113e34374e
34 changes: 34 additions & 0 deletions examples/zig-module/build.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
const std = @import("std");
const wws = @import("wws");

pub fn build(b: *std.Build) !void {
const target = wws.getTarget(b);
const optimize = b.standardOptimizeOption(.{});

const wws_dep = b.dependency("wws", .{});

const exe = b.addExecutable(.{
.name = "example",
.root_source_file = .{ .path = "src/main.zig" },
.target = target,
.optimize = optimize,
});
exe.wasi_exec_model = .reactor;
exe.root_module.addImport("wws", wws_dep.module("wws"));

b.installArtifact(exe);

const config =
\\name = "example"
\\version = "1"
\\[data]
\\[data.kv]
\\namespace = "example"
;
const wf = b.addWriteFiles();
const config_path = wf.add("example.toml", config);

const install_config = b.addInstallBinFile(config_path, "example.toml");

b.getInstallStep().dependOn(&install_config.step);
}
65 changes: 65 additions & 0 deletions examples/zig-module/build.zig.zon
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
.{
.name = "zig-build",
// This is a [Semantic Version](https://semver.org/).
// In a future version of Zig it will be used for package deduplication.
.version = "0.0.0",

// This field is optional.
// This is currently advisory only; Zig does not yet do anything
// with this value.
//.minimum_zig_version = "0.11.0",

// This field is optional.
// Each dependency must either provide a `url` and `hash`, or a `path`.
// `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
// Once all dependencies are fetched, `zig build` no longer requires
// internet connectivity.
.dependencies = .{
// See `zig fetch --save <url>` for a command-line interface for adding dependencies.
//.example = .{
// // When updating this field to a new URL, be sure to delete the corresponding
// // `hash`, otherwise you are communicating that you expect to find the old hash at
// // the new URL.
// .url = "https://example.com/foo.tar.gz",
//
// // This is computed from the file contents of the directory of files that is
// // obtained after fetching `url` and applying the inclusion rules given by
// // `paths`.
// //
// // This field is the source of truth; packages do not come from a `url`; they
// // come from a `hash`. `url` is just one of many possible mirrors for how to
// // obtain a package matching this `hash`.
// //
// // Uses the [multihash](https://multiformats.io/multihash/) format.
// .hash = "...",
//
// // When this is provided, the package is found in a directory relative to the
// // build root. In this case the package's hash is irrelevant and therefore not
// // computed. This field and `url` are mutually exclusive.
// .path = "foo",
//},
.wws = .{
.path = "../../kits/zig/wws",
},
},

// Specifies the set of files and directories that are included in this package.
// Only files and directories listed here are included in the `hash` that
// is computed for this package.
// Paths are relative to the build root. Use the empty string (`""`) to refer to
// the build root itself.
// A directory listed here means that all files within, recursively, are included.
.paths = .{
// This makes *all* files, recursively, included in this package. It is generally
// better to explicitly list the files and directories instead, to insure that
// fetching from tarballs, file system paths, and version control all result
// in the same contents hash.
"",
// For example...
//"build.zig",
//"build.zig.zon",
//"src",
//"LICENSE",
//"README.md",
},
}
67 changes: 67 additions & 0 deletions examples/zig-module/src/main.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
const std = @import("std");
const wws = @import("wws");

pub fn main() !void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
defer {
switch (gpa.deinit()) {
.ok => {},
.leak => {},
}
}
const allocator = gpa.allocator();

// Parse request from stdin
var event = try wws.parseStream(allocator, .{});
defer event.destroy();
const wws_request = event.request;

// Prepare response
var body = std.ArrayList(u8).init(allocator);
defer body.deinit();

try body.writer().print("{any} {s}\n", .{ wws_request.method, wws_request.url });

{
var it = wws_request.storage.iterator();
while (it.next()) |entry| {
try body.writer().print("kv.{s}: {s}\n", .{ entry.key_ptr.*, entry.value_ptr.* });
}
}

var headers = wws.Headers.init(allocator);
defer headers.deinit();
try headers.append("Content-Type", "text/plain");

var storage = std.StringHashMap([]const u8).init(allocator);
defer storage.deinit();
defer {
var it = storage.iterator();
while (it.next()) |entry| {
allocator.free(entry.value_ptr.*);
}
}

var counter: usize = if (wws_request.storage.get("counter")) |v| try std.fmt.parseInt(usize, v, 10) else 0;

// Increment counter, save the result to storage
counter += 1;

{
var buf = std.ArrayList(u8).init(allocator);
defer buf.deinit();
try buf.writer().print("{d}", .{counter});
try storage.put("counter", try buf.toOwnedSlice());
}

const response = try wws.formatResponse(allocator, .{
.data = body.items,
.status = 200,
.headers = &headers,
.storage = &storage,
});
defer allocator.free(response);

const stdout = std.io.getStdOut();
try stdout.writer().print("{s}", .{response});
}
44 changes: 44 additions & 0 deletions kits/zig/wws/build.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
const std = @import("std");

pub fn build(b: *std.Build) void {
_ = b.standardTargetOptions(.{});
_ = b.standardOptimizeOption(.{});

const module = b.addModule("wws", .{
.root_source_file = .{ .path = "src/wws.zig" },
.target = getTarget(b),
});

_ = module;
}

pub inline fn getTarget(b: *std.Build) std.Build.ResolvedTarget {
return b.resolveTargetQuery(.{
.cpu_arch = .wasm32,
.os_tag = .wasi,
});
}

pub const WwsLibOptions = struct {
name: []const u8,
root_source_file: std.Build.LazyPath,
optimize: std.builtin.OptimizeMode,
imports: []const std.Build.Module.Import = &.{},
};

pub fn addExecutable(b: *std.Build, options: WwsLibOptions) *std.Build.Step.Compile {
const exe = b.addExecutable(.{
.name = options.name,
.root_source_file = options.root_source_file,
.target = getTarget(b),
.optimize = options.optimize,
});

exe.wasi_exec_model = .reactor;

for (options.imports) |import| {
exe.root_module.addImport(import.name, import.module);
}

return exe;
}
62 changes: 62 additions & 0 deletions kits/zig/wws/build.zig.zon
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
.{
.name = "wws",
// This is a [Semantic Version](https://semver.org/).
// In a future version of Zig it will be used for package deduplication.
.version = "0.0.0",

// This field is optional.
// This is currently advisory only; Zig does not yet do anything
// with this value.
//.minimum_zig_version = "0.11.0",

// This field is optional.
// Each dependency must either provide a `url` and `hash`, or a `path`.
// `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
// Once all dependencies are fetched, `zig build` no longer requires
// internet connectivity.
.dependencies = .{
// See `zig fetch --save <url>` for a command-line interface for adding dependencies.
//.example = .{
// // When updating this field to a new URL, be sure to delete the corresponding
// // `hash`, otherwise you are communicating that you expect to find the old hash at
// // the new URL.
// .url = "https://example.com/foo.tar.gz",
//
// // This is computed from the file contents of the directory of files that is
// // obtained after fetching `url` and applying the inclusion rules given by
// // `paths`.
// //
// // This field is the source of truth; packages do not come from a `url`; they
// // come from a `hash`. `url` is just one of many possible mirrors for how to
// // obtain a package matching this `hash`.
// //
// // Uses the [multihash](https://multiformats.io/multihash/) format.
// .hash = "...",
//
// // When this is provided, the package is found in a directory relative to the
// // build root. In this case the package's hash is irrelevant and therefore not
// // computed. This field and `url` are mutually exclusive.
// .path = "foo",
//},
},

// Specifies the set of files and directories that are included in this package.
// Only files and directories listed here are included in the `hash` that
// is computed for this package.
// Paths are relative to the build root. Use the empty string (`""`) to refer to
// the build root itself.
// A directory listed here means that all files within, recursively, are included.
.paths = .{
// This makes *all* files, recursively, included in this package. It is generally
// better to explicitly list the files and directories instead, to insure that
// fetching from tarballs, file system paths, and version control all result
// in the same contents hash.
"",
// For example...
//"build.zig",
//"build.zig.zon",
//"src",
//"LICENSE",
//"README.md",
},
}
242 changes: 242 additions & 0 deletions kits/zig/wws/src/wws.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,242 @@
const std = @import("std");

pub const Method = std.http.Method;
pub const Headers = std.http.Headers;

pub const Request = struct {
url: []const u8,
body: []const u8,
method: Method,
headers: *Headers,
storage: *std.StringHashMap([]const u8),
params: *std.StringHashMap([]const u8),
};

const InputStreamType = enum {
stdin,
};

const InputStream = union(InputStreamType) {
stdin: void,
};

const ParseStreamError = error{ OutOfMemory, UnknownError, MalformedRequestObject };

const ParseStreamOptions = struct {
input_stream: InputStream = .stdin,
};

const ParseStreamResult = struct {
allocator: std.mem.Allocator,
request: *Request,

pub fn destroy(self: *ParseStreamResult) void {
{
var it = self.request.params.iterator();
while (it.next()) |*entry| {
self.allocator.free(entry.key_ptr.*);
self.allocator.free(entry.value_ptr.*);
}
self.request.params.deinit();
self.allocator.destroy(self.request.params);
}

{
var it = self.request.storage.iterator();
while (it.next()) |*entry| {
self.allocator.free(entry.key_ptr.*);
self.allocator.free(entry.value_ptr.*);
}
self.request.storage.deinit();
self.allocator.destroy(self.request.storage);
}

self.request.headers.deinit();
self.allocator.destroy(self.request.headers);

self.allocator.free(self.request.url);
self.allocator.free(self.request.body);

self.allocator.destroy(self.request);
}
};

/// Caller owns the memory
pub fn parseStream(allocator: std.mem.Allocator, options: ParseStreamOptions) ParseStreamError!ParseStreamResult {
const stream = switch (options.input_stream) {
.stdin => std.io.getStdIn().reader(),
};

var input = std.ArrayList(u8).init(allocator);
defer input.deinit();

stream.readAllArrayList(&input, std.math.maxInt(usize)) catch return ParseStreamError.UnknownError;

var json = std.json.parseFromSlice(std.json.Value, allocator, input.items, .{}) catch return ParseStreamError.UnknownError;
defer json.deinit();

const r = try allocator.create(Request);

switch (json.value) {
.object => |root| {
switch (root.get("url") orelse return ParseStreamError.MalformedRequestObject) {
.string => |s| {
r.*.url = try allocator.dupe(u8, s);
},
else => return ParseStreamError.MalformedRequestObject,
}

switch (root.get("method") orelse return ParseStreamError.MalformedRequestObject) {
.string => |s| {
r.*.method = @enumFromInt(Method.parse(s));
},
else => return ParseStreamError.MalformedRequestObject,
}

switch (root.get("headers") orelse return ParseStreamError.MalformedRequestObject) {
.object => |o| {
const headers = try allocator.create(Headers);
headers.* = Headers.init(allocator);
errdefer headers.deinit();

var it = o.iterator();
while (it.next()) |kv| {
switch (kv.value_ptr.*) {
.string => |v| {
try headers.append(kv.key_ptr.*, v);
},
else => return ParseStreamError.MalformedRequestObject,
}
}

r.*.headers = headers;
},
else => return ParseStreamError.MalformedRequestObject,
}

switch (root.get("body") orelse return ParseStreamError.MalformedRequestObject) {
.string => |s| {
r.*.body = try allocator.dupe(u8, s);
},
else => return ParseStreamError.MalformedRequestObject,
}

switch (root.get("kv") orelse return ParseStreamError.MalformedRequestObject) {
.object => |o| {
const storage = try allocator.create(std.StringHashMap([]const u8));
storage.* = std.StringHashMap([]const u8).init(allocator);
errdefer storage.deinit();

var it = o.iterator();
while (it.next()) |kv| {
switch (kv.value_ptr.*) {
.string => |v| {
const owned_key = try allocator.dupe(u8, kv.key_ptr.*);
errdefer allocator.free(owned_key);

const owned_value = try allocator.dupe(u8, v);
errdefer allocator.free(owned_value);

try storage.put(owned_key, owned_value);
},
else => return ParseStreamError.MalformedRequestObject,
}
}

r.*.storage = storage;
},
else => return ParseStreamError.MalformedRequestObject,
}

switch (root.get("params") orelse return ParseStreamError.MalformedRequestObject) {
.object => |o| {
const params = try allocator.create(std.StringHashMap([]const u8));
params.* = std.StringHashMap([]const u8).init(allocator);
errdefer params.deinit();

var it = o.iterator();
while (it.next()) |kv| {
switch (kv.value_ptr.*) {
.string => |v| {
const owned_key = try allocator.dupe(u8, kv.key_ptr.*);
errdefer allocator.free(owned_key);

const owned_value = try allocator.dupe(u8, v);
errdefer allocator.free(owned_value);

try params.put(owned_key, owned_value);
},
else => return ParseStreamError.MalformedRequestObject,
}
}

r.*.params = params;
},
else => return ParseStreamError.MalformedRequestObject,
}
},
else => return ParseStreamError.MalformedRequestObject,
}

return .{
.allocator = allocator,
.request = r,
};
}

const FormatResponseOptions = struct {
data: []const u8,
status: usize,
headers: *Headers,
storage: *std.StringHashMap([]const u8),
};

const FormatResponseError = error{
UnknownError,
OutOfMemory,
};

pub fn formatResponse(allocator: std.mem.Allocator, options: FormatResponseOptions) FormatResponseError![]const u8 {
var buf = std.ArrayList(u8).init(allocator);
defer buf.deinit();

var w = std.json.writeStream(buf.writer(), .{ .whitespace = .minified });
{
try w.beginObject();

try w.objectField("data");
try w.write(std.json.Value{ .string = options.data });

try w.objectField("status");
try w.write(options.status);

{
var o = std.json.ObjectMap.init(allocator);
defer o.deinit();

for (options.headers.list.items) |entry| {
try o.put(entry.name, .{ .string = entry.value });
}

try w.objectField("headers");
try w.write(std.json.Value{ .object = o });
}

{
var o = std.json.ObjectMap.init(allocator);
defer o.deinit();

var it = options.storage.iterator();
while (it.next()) |entry| {
try o.put(entry.key_ptr.*, .{ .string = entry.value_ptr.* });
}

try w.objectField("kv");
try w.write(std.json.Value{ .object = o });
}

try w.endObject();
}

return buf.toOwnedSlice() catch FormatResponseError.UnknownError;
}