Refactor makesyscalls.lua into a library

* main.lua replicates the functionality of makesyscalls.lua
* Individual files are generated by their associated module
  * Modules can be called as standalone scripts to generate a specific
    file
* Data and procedures are performed by objects instead of procedual code
* Bitmasks are replaced by declarative types
* Temporary files are no longer produced, writing is stored in memory
* Comments provide explanation to functions and semantics

Google Summer of Code 2024 Final Work Product

Sponsored by:    Google (GSoC 24)
Co-authored-by: Warner Losh <[email protected]>
Co-authored-by: Brooks Davis <[email protected]>
Pull Request:	freebsd#1362

sys/tools/syscalls/README.md

@@ -0,0 +1,49 @@
+# System call creation library
+Parses `syscalls.master` and packages information into objects with methods.
+Modules reproduce the previous file auto-generation of `makesyscalls.lua`.
+
+We generally assume that this script will be run by flua, however we've
+carefully crafted modules for it that mimic interfaces provided by modules
+available in ports.  Currently, this script is compatible with lua from
+ports along with the compatible luafilesystem and lua-posix modules.
+
+## Usage
+`main.lua` generates all files.
+Files are associated with their respective modules, and modules can be run as
+standalone scripts to generate specific files.
+
+### Examples
+**All files:**
+`# /usr/libexec/flua /usr/src/sys/tools/syscalls/main.lua /usr/src/sys/kern/syscalls.master`
+<br>
+**syscalls.h:**
+`# /usr/libexec/flua /usr/src/sys/tools/syscalls/modules/syscalls.h /usr/src/sys/kern/syscalls.master`
+
+## Organization
+* `root`
+  * `main.lua` - Main entry point that calls all scripts.
+  * `config.lua` - Contains the global configuration table and associated
+                   configuration functions.
+
+  * `core` (Core Classes)
+    * `syscall.lua` - Packages each system call entry from `syscalls.master`
+                      into a system call object.
+    * `scarg.lua` - Packages each argument for the system call into an argument
+                    object.
+    * `scret.lua` - An object for the return value of the system call.
+    * `freebsd-syscall.lua` - Contains the master system call table after
+                              processing.
+
+  * `scripts`
+    * `init_sysent.lua` - Generates `init_sysent.c`.
+    * `libsys_h.lua` - Generates `lib/libsys/_libsys.h`.
+    * `syscall_h.lua` - Generates `syscall.h`.
+    * `syscall_mk.lua` - Generates `syscall.mk`.
+    * `syscalls.lua` - Generates `syscalls.c`.
+    * `syscalls_map.lua` - Generates `lib/libsys/syscalls.map`.
+    * `sysproto_h.lua` - Generates `sysproto.h`.
+    * `systrace_args.lua` - Generates `systrace_args.c`.
+
+  * `tools`
+    * `util.lua` - Contains utility functions.
+    * `generator.lua` - Handles file generation for the library.

sys/tools/syscalls/config.lua

@@ -0,0 +1,302 @@
+--
+-- SPDX-License-Identifier: BSD-2-Clause
+--
+-- Copyright (c) 2021-2024 SRI International
+-- Copyright (c) 2024 Tyler Baxter <[email protected]>
+-- Copyright (c) 2023 Warner Losh <[email protected]>
+-- Copyright (c) 2019 Kyle Evans <[email protected]>
+--
+
+--
+-- Code to read in the config file that drives this. Since we inherit from the
+-- FreeBSD makesyscall.sh legacy, all config is done through a config file that
+-- sets a number of variables (as noted below); it used to be a .sh file that
+-- was sourced in. This dodges the need to write a command line parser.
+--
+
+local util = require("tools.util")
+
+--
+-- Global config map.
+-- Default configuration is native. Any of these may get replaced by an
+-- optionally specified configuration file.
+--
+local config = {
+	sysnames = "syscalls.c",
+	syshdr = "../sys/syscall.h",
+	sysmk = "/dev/null",
+	syssw = "init_sysent.c",
+	systrace = "systrace_args.c",
+	sysproto = "../sys/sysproto.h",
+	libsysmap = "/dev/null",
+	libsys_h = "/dev/null",
+	sysproto_h = "_SYS_SYSPROTO_H_",
+	syscallprefix = "SYS_",
+	switchname = "sysent",
+	namesname = "syscallnames",
+	abi_flags = {},
+	abi_func_prefix = "",
+	abi_type_suffix = "",
+	abi_long = "long",
+	abi_u_long = "u_long",
+	abi_semid_t = "semid_t",
+	abi_size_t = "size_t",
+	abi_ptr_array_t = "",
+	abi_headers = "",
+	abi_intptr_t = "intptr_t",
+	ptr_intptr_t_cast = "intptr_t",
+	obsol = {},
+	unimpl = {},
+	capabilities_conf = "capabilities.conf",
+	compat_set = "native",
+	mincompat = 0,
+	capenabled = {},
+	-- System calls that require ABI-specific handling
+	syscall_abi_change = {},
+	-- System calls that appear to require handling, but don't
+	syscall_no_abi_change = {},
+	-- Keep track of modifications if there are.
+	modifications = {},
+	-- Stores compat_sets from syscalls.conf; config.mergeCompat() instantiates.
+	compat_options = {},
+}
+
+--
+-- For each entry, the ABI flag is the key. One may also optionally provide an
+-- expr, which are contained in an array associated with each key; expr gets
+-- applied to each argument type to indicate whether this argument is subject to
+-- ABI change given the configured flags.
+--
+config.known_abi_flags = {
+	long_size = {
+		"_Contains[a-z_]*_long_",
+		"^long [a-z0-9_]+$",
+		"long [*]",
+		"size_t [*]",
+		-- semid_t is not included because it is only used
+		-- as an argument or written out individually and
+		-- said writes are handled by the ksem framework.
+		-- Technically a sign-extension issue exists for
+		-- arguments, but because semid_t is actually a file
+		-- descriptor negative 32-bit values are invalid
+		-- regardless of sign-extension.
+	},
+	time_t_size = {
+		"_Contains[a-z_]*_timet_",
+	},
+	pointer_args = {
+		-- no expr
+	},
+	pointer_size = {
+		"_Contains[a-z_]*_ptr_",
+		"[*][*]",
+	},
+	pair_64bit = {
+		"^dev_t[ ]*$",
+		"^id_t[ ]*$",
+		"^off_t[ ]*$",
+	},
+}
+
+-- All compat option entries should have five entries:
+--	definition: The preprocessor macro that will be set for this
+--	compatlevel: The level this compatibility should be included at.  This
+--		generally represents the version of FreeBSD that it is compatible
+--		with, but ultimately it's just the level of mincompat in which it's
+--		included.
+--	flag: The name of the flag in syscalls.master.
+--	prefix: The prefix to use for _args and syscall prototype.  This will be
+--		used as-is, without "_" or any other character appended.
+--	descr: The description of this compat option in init_sysent.c comments.
+-- The special "stdcompat" entry will cause the other five to be autogenerated.
+local compat_option_sets = {
+	native = {
+		{
+			definition = "COMPAT_43",
+			compatlevel = 3,
+			flag = "COMPAT",
+			prefix = "o",
+			descr = "old",
+		},
+		{ stdcompat = "FREEBSD4" },
+		{ stdcompat = "FREEBSD6" },
+		{ stdcompat = "FREEBSD7" },
+		{ stdcompat = "FREEBSD10" },
+		{ stdcompat = "FREEBSD11" },
+		{ stdcompat = "FREEBSD12" },
+		{ stdcompat = "FREEBSD13" },
+		{ stdcompat = "FREEBSD14" },
+	},
+}
+
+-- config looks like a shell script; in fact, the previous makesyscalls.sh
+-- script actually sourced it in.  It had a pretty common format, so we should
+-- be fine to make various assumptions
+function config.process(file)
+	local cfg = {}
+	local comment_line_expr = "^%s*#.*"
+	-- We capture any whitespace padding here so we can easily advance to
+	-- the end of the line as needed to check for any trailing bogus bits.
+	-- Alternatively, we could drop the whitespace and instead try to
+	-- use a pattern to strip out the meaty part of the line, but then we
+	-- would need to sanitize the line for potentially special characters.
+	local line_expr = "^([%w%p]+%s*)=(%s*[`\"]?[^\"`]*[`\"]?)"
+
+	if not file then
+		return nil, "No file given"
+	end
+
+	local fh = assert(io.open(file))
+
+	for nextline in fh:lines() do
+		-- Strip any whole-line comments
+		nextline = nextline:gsub(comment_line_expr, "")
+		-- Parse it into key, value pairs
+		local key, value = nextline:match(line_expr)
+		if key ~= nil and value ~= nil then
+			local kvp = key .. "=" .. value
+			key = util.trim(key)
+			value = util.trim(value)
+			local delim = value:sub(1,1)
+			if delim == '"' then
+				local trailing_context
+
+				-- Strip off the key/value part
+				trailing_context = nextline:sub(kvp:len() + 1)
+				-- Strip off any trailing comment
+				trailing_context = trailing_context:gsub("#.*$",
+				    "")
+				-- Strip off leading/trailing whitespace
+				trailing_context = util.trim(trailing_context)
+				if trailing_context ~= "" then
+					print(trailing_context)
+					util.abort(1,
+					    "Malformed line: " .. nextline)
+				end
+
+				value = util.trim(value, delim)
+			else
+				-- Strip off potential comments
+				value = value:gsub("#.*$", "")
+				-- Strip off any padding whitespace
+				value = util.trim(value)
+				if value:match("%s") then
+					util.abort(1,
+					    "Malformed config line: " ..
+					    nextline)
+				end
+			end
+			cfg[key] = value
+		elseif not nextline:match("^%s*$") then
+			-- Make sure format violations don't get overlooked
+			-- here, but ignore blank lines.  Comments are already
+			-- stripped above.
+			util.abort(1, "Malformed config line: " .. nextline)
+		end
+	end
+
+	assert(fh:close())
+	return cfg
+end
+
+-- Merges processed configuration file into the global config map (see above),
+-- or returns NIL and a message.
+function config.merge(fh)
+	if fh ~= nil then
+		local res = assert(config.process(fh))
+
+		for k, v in pairs(res) do
+			if v ~= config[k] then
+				-- Handling of string lists:
+				if k:find("abi_flags") then
+					-- Match for pipe, that's how abi_flags
+					-- is formatted.
+					config[k] = util.setFromString(v, "[^|]+")
+				elseif k:find("capenabled") or
+				    k:find("syscall_abi_change") or
+				    k:find("syscall_no_abi_change") or
+				    k:find("obsol") or
+				    k:find("unimpl") then
+					-- Match for space, that's how these
+					-- are formatted.
+					config[k] = util.setFromString(v, "[^ ]+")
+				else
+					config[k] = v
+				end
+				-- Construct config modified table as config
+				-- is processed.
+				config.modifications[k] = true
+			else
+				-- config wasn't modified
+				config.modifications[k] = false
+			end
+		end
+	end
+end
+
+-- Returns TRUE if there are ABI changes from native for the provided ABI flag.
+function config.abiChanges(name)
+	if config.known_abi_flags[name] == nil then
+		util.abort(1, "abi_changes: unknown flag: " .. name)
+	end
+	return config.abi_flags[name] ~= nil
+end
+
+-- Instantiates config.compat_options.
+function config.mergeCompat()
+	if config.compat_set ~= "" then
+		if not compat_option_sets[config.compat_set] then
+			util.abort(1, "Undefined compat set: " ..
+			    config.compat_set)
+		end
+
+		config.compat_options = compat_option_sets[config.compat_set]
+	end
+end
+
+-- Parses the provided capabilities.conf. Returns a string (comma separated
+-- list) as its formatted in capabilities.conf.
+local function grabCapenabled(file, open_fail_ok)
+	local capentries = {}
+	local commentExpr = "#.*"
+
+	if file == nil then
+		return nil, "No file given"
+	end
+
+	local fh, msg, errno = io.open(file)
+	if fh == nil then
+		if not open_fail_ok then
+			util.abort(errno, msg)
+		end
+		return nil, msg
+	end
+
+	for nextline in fh:lines() do
+		-- Strip any comments
+		nextline = nextline:gsub(commentExpr, "")
+		if nextline ~= "" then
+			capentries[nextline] = true
+		end
+	end
+
+	assert(fh:close())
+	return capentries
+end
+
+-- Merge capability (Capsicum) configuration into the global config.
+function config.mergeCapability()
+	-- We ignore errors here if we're relying on the default configuration.
+	if not config.modifications.capenabled then
+		config.capenabled = grabCapenabled(config.capabilities_conf,
+		    config.modifications.capabilities_conf == nil)
+	elseif config.capenabled ~= "" then
+		-- We have a comma separated list from the format of
+		-- capabilities.conf, split it into a set with boolean values
+		-- for each key.
+		config.capenabled = util.setFromString(config.capenabled,
+		    "[^,]+")
+	end
+end
+
+return config