Redesign of package os

[gingerBill]

Redesign of package os.

This will need to have a minimal OS interface. e.g. on Windows, it should have its own miniature "core:sys/win32" rather than importing that.

The package's interface needs to be the same across all OSes.

The following code is just an example of the interface that I want, and not the final design.

package os

Error :: enum {
	None,
	Invalid_Argument,
	Permission_Denied,
	Exist,
	Not_Exist,
	Closed,
	...
	// This could be a little more complicated
}

File_Flag :: enum {
	Read_Only,
	Write_Only,
	Read_Write,
	Create,
	Excl,
	Truncate,
	Append,
	Sync,
	...
}
File_Flags :: bit_set[File_Flag; u32];


File_Mode :: distinct u32;

stdin, stdout, stderr: ^File;

args: []string;

change_directory :: proc(dir: string) -> Error {}
change_mode :: proc(name: string, mode: File_Mode) -> Error {}
change_ownership :: proc(name: string, uid, gid: int) -> Error {}
change_ownership_link :: proc(name: string, uid, gid: int) -> Error {}
change_times :: proc(name: string, atime, mtime: time.Time) -> Error {}

clear_env :: proc() {}
environ :: proc(allocator := context.allocator) -> []string {}
get_env :: proc(key: string, allocator := context.allocator) -> string {} 
lookup_env :: proc(key: string, allocator := context.allocator) -> (string, bool) {} 
set_env :: proc(key, value: string) -> Error {}
unset_env :: proc(key: string) -> Error {}

executable :: proc() -> (string, Error) {}

exit :: proc(code: int) {}

egid :: proc() -> int {} // -1 on Windows
euid :: proc() -> int {} // -1 on Windows
gid :: proc() -> int {} // -1 on Windows
pid :: proc() -> int {} 
ppid :: proc() -> int {} 
uid :: proc() -> int {} // -1 on Windows

current_thread_id :: proc() -> int {}

page_size :: proc() -> int {}

working_directory :: proc(allocator := context.allocator) -> (dir: string, err: Error) {}
host_name :: proc(allocator := context.allocator) -> (dir: string, err: Error) {}

link :: proc(old_name, new_name: string) -> Error {}
symlink :: proc(old_name, new_name: string) -> Error {}

make_directory :: proc(name: string, perm: File_Mode) -> Error {}
make_directory_all :: proc(path: string, perm: File_Mode) -> Error {}


pipe :: proc() -> (r, w: ^File, err: Error) {}

read_link :: proc(name: string, allocator := context.allocator) -> (string, Error) {}

remove :: proc(name: string) -> Error {}
remove_all :: proc(path: string) -> Error {}
rename :: proc(old_path, new_path: string) -> Error {}

temp_directory :: proc() -> string {}
user_cache_directory :: proc() -> (string, Error) {}
user_home_directory :: proc() -> (string, Error) {}

truncate :: proc(name: string, size: i64) -> Error {}


// Should a file be a type or just a handle?
// If not, probably explicitly overload procedures to work with either ^File or a handle
File :: struct {
	...
}

File_Info :: struct {
	name:            string,
	size:            i64,
	mode:            File_Mode,
	mod_time:        time.Time,
	is_directory:    bool,
	underlying_data: rawptr,
}

create :: proc(name: string) -> (^File, Error) {}
new_file :: proc(fd: uintptr, name: string) -> ^File {}

open :: proc(name: string, flags := File_Flags{.Read_Only}, perm := File_Mode(0)) -> (^File, Error) {}
close :: proc(f: ^File) -> Error {}

// These could be added to a procedure group
file_change_directory :: proc(f: ^File) -> Error {}
file_change_mode :: proc(f: ^File, mode: File_Mode) -> Error {}
file_change_ownership :: proc(f: ^File, uid, gid: int) -> Error {}
file_change_ownership_link :: proc(f: ^File, uid, gid: int) -> Error {}

fd :: proc(f: ^File) -> uintptr {}
file_name :: proc(f: ^File) -> uintptr {}

read :: proc(f: ^File, b: []byte) -> (n: int, err: Error) {}
read_at :: proc(f: ^File, b: []byte, offset: i64) -> (n: int, err: Error) {}
read_ptr :: proc(f: ^File, data: rawptr, len: int) -> (n: int, err: Error) {}

write :: proc(f: ^File, b: []byte) -> (n: int, err: Error) {}
write_at :: proc(f: ^File, b: []byte, offset: i64) -> (n: int, err: Error) {}
write_ptr :: proc(f: ^File, data: rawptr, len: int) -> (n: int, err: Error) {}
write_string :: proc(f: ^File, s: string) -> (n: int, err: Error) {}
write_byte :: proc(f: ^File, b: byte) -> (n: int, err: Error) {}
write_rune :: proc(f: ^File, r: rune) -> (n: int, err: Error) {}
write_encoded_rune :: proc(f: ^File, r: rune) -> (n: int, err: Error) {}


seek :: proc(f: ^File, offset: i64, whence: int) -> (ret: i64, err: Error) {}
sync :: proc(f: ^File) -> Error {}
truncate :: proc(f: ^File, size: i64) -> Error {}

file_stat :: proc(f: ^File) -> (File_Info, Error) {}
file_stat_link :: proc(f: ^File) -> (File_Info, Error) {}
read_directory :: proc(f: ^File, n: int) -> ([]File_Info, Error) {}
read_directory_names :: proc(f: ^File, n: int) -> (names: []string, err: Error) {}

stat :: proc(name: string) -> (File_Info, Error) {}
stat_link :: proc(name: string) -> (File_Info, Error) {}

read_entire_file  :: proc(name: string) -> (data: []byte, err: Error) {}
write_entire_file :: proc(name: string, data: []byte, truncate := true) -> (err: Error) {} 

file_size_from_path :: proc(name: string) -> (i64, Error) {}
file_size :: proc(f: ^File) -> (i64, Error) {}


Process :: struct {
	pid: int,
	...
}

Process_Attributes :: struct {
	...
}

Process_State :: struct {
	exit_code: int,
	exited: bool,
	pid: int,
	success: bool,
	system_time: time.Duration,
	user_time: time.Duration,
	underlying_data: rawptr,
}

find_process :: proc(pid: int) -> (^Process, Error) {}
start_process :: proc(name: string, argv: []string, attr: ^Process_Attributes) -> (^Process, Error) {}
kill_process :: proc(p: ^Process) -> Error {}
release_process :: proc(p: ^Process) -> Error {}
signal_process :: proc(p: ^Process, signal: Signal) -> Error {}
wait_process :: proc(p: ^Process) -> (Process_State, Error) {}

Signal :: struct {
	...
}


heap_allocator :: proc() -> mem.Allocator {}
virtual_allocator :: proc() -> mem.Allocator {}

heap_alloc :: proc(size: int) -> rawptr {}
heap_resize :: proc(ptr: rawptr, new_size: int) -> rawptr {}
heap_free :: proc(ptr: rawptr) -> rawptr {}

[gingerBill]

Should "files" be represented by a custom data structure or a handle?

[Lunatoid]

I feel like having the filename, file size and other metadata in a struct might be useful to have on-hand.

If the metadata is unnecessary for the function it could just accept the handle on the struct. (some error returns left out for clarity)

file_info: os.File_Info = os.open(path);
os.read(file_info.handle, buffer);
os.close(file_info.handle);

[gingerBill]

I'm not sure what the best approach is. If you need the file info, you can ask for it from the handle, but having a semi-opaque struct means you can store extra information in the struct for internal issue.

[gingerBill]

So it's an interesting design decision, the reason being is that the OS will be storing a lot of the file information internally already. And things like caching the long name for the file is only useful on Windows as it uses 16-bit wide chars for everything.

[Lunatoid]

The handle could be an semi-opaque struct and then functions like os.get_file_size(...) could either get it from the semi-opaque struct if it stores it, and if it doesn't, it could make an API call to the operating system to get it.

That way any relevant caching can be done for each OS while still supplying uniform functions and behavior.

[gingerBill]

The other question is whether to use short names or long names for things e.g. chdir or change_directory.

I prefer the latter but I guess most people are more familiar with chdir.

[gingerBill]

I think something like this is the best option:

package os

Handle :: distinct rawptr; // or 'uintptr' whatever is better
File :: distinct Handle;

It solves a lot of the TOCTOU issues.

[Lunatoid]

I prefer the latter as well, the thing is is that you need to be consistent if you decide on either of them. You can't make os.change_directory and then also have os.mkdir since that would be inconsistent naming.

Personally when I see the former I think of the console command instead of an actual API call but that's just me.

[gingerBill]

@Lunatoid Of course. Consistency is key here.

[Lunatoid]

So in the example you posted, the handles/files are pointers to OS specific structs or is it just a "typedef" of e.g. a Win32 HANDLE?

[gingerBill]

In a sense, yes.

[gingerBill]

Current branch for work: https://github.com/odin-lang/Odin/tree/dev-pkg-os/core/os_v2

[nakst]

Here's a list of things that I'd expect in a platform layer (based on Essence's API):

• Processes
  • Creation (executable path, environment block, command line arguments)
  • Open handle from PID
  • List process (executable path, PID, memory/CPU usage)
  • Terminate (with/without error)
  • Pause/resume
  • Get PID from handle
  • Get exit code
• Threads
  • Creation
  • Set/get thread local storage pointer
  • Terminate
  • Open from TID
  • Get TID from handle
• Filesystem
  • Open node (returning file size/number files in directory at time of opening)
    • Access flags: read (none, block, access, exclusive), write (none, block, access, exclusive)
    • Creation mode: fail if found, otherwise create new file; fail if not found; create if not found; truncate if found, create if not found.
    • Creating leading directories
    • Parsing '.' and '..'
  • Read/write entire files
  • Read/write/resize synchronously
  • Read/write/resize asynchronously
    • Get progress of IO request
    • Cancel IO request
  • Get file size/number of files in directory
  • Delete/move node
  • Get user home folder
• Synchronisation
  • Events (auto/manual reset)
    • Set/reset
    • Poll
    • Wait on single event
    • Wait on multiple events
  • Mutexes
    • Create/acquire/release
  • Spinlocks
    • Create/acquire/release
  • Timers
    • Create
    • Set to fire in n milliseconds
    • Set completion callback pointer/argument
• Virtual memory
  • Allocate/free
  • Open shared memory (fail if found, fail if not found; read-only/read-write/read-execute/read-write-execute)
  • Open handle to shared memory for other process (read-only/read-write)
• Misc
  • Close handle
  • Get system information (kernel object counts, memory usage, %CPU usage)
  • Spin for n microseconds
  • Wait for n milliseconds
  • Output debug string

[gingerBill]

Processes and the filesystem aspects are the main ideas behind package os. Threading and virtual memory need to be separate packages.

[Platin21]

Would consider these stats/infos also as nice to have in a os independent way:

• max ram (installed as dimm)
• cpu count
• network speed (if that doesn't go to far and would better live in package net)
• list files in folder (recursive/non recursive)
• get names of files in folder (recursive/non recursive)
• count of folders in folder (recursive/non recursive)
• get env (Environment variables)
• memory mapping
• print to system error log / warning

And for proc names i would use the long names not the console names like mkdir, …

[hazeycode]

What about POSIX compliant stuffs i.e. sharing POSIX stuff across compliant OSs? Much of the linux and macOS implementation will be very similar and identical in places

[gingerBill]

I am not so interested in POSIX compliance. Especially since Odin's constructs will not map directly to the POSIX C interfaces.

[Platin21]

Some of these are redundant?
egid,euid,gid,pid,ppid,uid
Or did i miss something.

[Platin21]

Should we still make the heap and virtualalloc stuff?
As i think a better place is mem.
Same for syslog would put it in core:log

[gingerBill]

That's a good question. Maybe the normal stuff should be in os but allocators be in mem

[Platin21]

Current Proposed read functions:
https://gist.github.com/Platin21/1af219538072394a05826692648b732e

I was so free and added these:
start_process :: proc(name: string, args: Args) -> (bool, Pid)
terminate_process :: proc(pid: Pid) -> (bool,c.int)
virtual_cpu_count :: proc() -> u16

And here was the first proposed design:
https://gist.github.com/Platin21/5c4972d7b333f7d3d9940fe9fd7a75da

[Platin21]

The other thing is what about memory for some of these functions there is more than just temporary memory needed for example the process list.

How should we deal with that?

[Platin21]

Also what about errors?
Should probably better discuss this on discord but conversation often drifts away from main point.

[Platin21]

I tried to discuss this several times there it always drifted into should we even have os ..

[rytc]

I did some work on the Linux file API for v2, let me know if this is inline with expectations

Branch: https://github.com/rytc/Odin/tree/linux_osv2
https://github.com/rytc/Odin/blob/linux_osv2/core/os/os2/file_linux.odin

[Platin21]

I’d move the syscalls to sys and import them

[gingerBill]

I think I may also change how the Error system works to make it a union of many different enums if necessary.

e.g.

Error :: enum {
    FS_Error,
    Pipe_Error,
    System_Error,
}

FS_Error :: enum i32 { ... };
Pipe_Error :: enum i32 { ... };
System_Error :: distinct i32;

[Platin21]

Would probably also make sense to compare the currently supported os’es on the error values.

[Platin21]

Has anyone looked at os2? I have found it really ugly that it now requires one todo allocations for opening a file? For me this feels a bit crazy.

[Platin21]

A other thing is if some of the proposed procs for processes should go in there own package? As the first proposal for a function set included these procs.

Idea: core:process or core:os/process or exec (maybe a bit too narrow for what you can do)