Osquerybeat: Implement host_users, host_groups, host_processes tables…

… as a part of our osquery_extension. (#28434) (#28553)

* Osquerybeat: Implement host_users, host_groups, host_processes tables as a part of our osquery_extension.

* The expectation is that the /etc/passwd,  /etc/group,  /proc
are avaiable in the container under /hostfs as: /hostfs/etc/passwd, /hostfs/etc/group, /hostfs/proc.
The ELASTIC_OSQUERY_HOSTFS environment variable allows to change the default.
* The on_disk column is always set to -1

* Remove erroneous comment line

(cherry picked from commit ec8e051)

Co-authored-by: Aleksandr Maus <[email protected]>

x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/common.go

@@ -0,0 +1,92 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package hostfs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+const (
+	defaultMount      = "/hostfs"
+	envHostFSOverride = "ELASTIC_OSQUERY_HOSTFS" // Allows to override the mount point for hostfs, default is /hostfs
+)
+
+var (
+	ErrMissingField     = errors.New("missing/invalid field")
+	ErrInvalidFieldType = errors.New("invalid field type")
+)
+
+type ColumnType int
+
+const (
+	ColumnTypeString ColumnType = iota
+	ColumnTypeInt
+	ColumnTypeUint
+)
+
+func (c ColumnType) String() string {
+	return [...]string{"string", "int64", "uint64"}[c]
+}
+
+type ColumnInfo struct {
+	IndexFrom int
+	Name      string
+	Type      ColumnType
+	Optional  bool
+}
+
+func GetPath(fp string) string {
+	// Check the environment variable for override, otherwise use /hostfs as the mount root
+	mountRoot := os.Getenv(envHostFSOverride)
+	if mountRoot == "" {
+		mountRoot = defaultMount
+	}
+	return filepath.Join(mountRoot, fp)
+}
+
+type StringMap map[string]string
+
+func (m StringMap) Set(fields []string, col ColumnInfo) error {
+	if col.IndexFrom >= len(fields) {
+		if !col.Optional {
+			return fmt.Errorf("failed to read field at index: %d, when total number of fields is: %d, err: %w", col.IndexFrom, len(fields), ErrMissingField)
+		}
+		m[col.Name] = ""
+		return nil
+	}
+
+	var err error
+
+	sval := fields[col.IndexFrom]
+	// Check that it is convertable to int type
+	switch col.Type {
+	case ColumnTypeUint:
+		// For unsigned values (Apple) the number is parsed as signed int32 then converted to unsigned.
+		// This is consistent with osquery `users` table data on Mac OS.
+		// osquery> select * from users;
+		// +------------+------------+------------+------------+------------------------+-------------------------------------------------+-------------------------------+------------------+--------------------------------------+-----------+
+		// | uid        | gid        | uid_signed | gid_signed | username               | description                                     | directory                     | shell            | uuid                                 | is_hidden |
+		// +------------+------------+------------+------------+------------------------+-------------------------------------------------+-------------------------------+------------------+--------------------------------------+-----------+
+		// | 229        | 4294967294 | 229        | -2         | _avbdeviced            | Ethernet AVB Device Daemon                      | /var/empty                    | /usr/bin/false   | FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000E5 | 0         |
+		v, err := strconv.ParseInt(sval, 10, 32)
+		if err == nil {
+			n := uint32(v)
+			sval = strconv.FormatUint(uint64(n), 10)
+		}
+	case ColumnTypeInt:
+		_, err = strconv.ParseInt(sval, 10, 64)
+	}
+
+	if err != nil {
+		return fmt.Errorf("invalid field type at index: %d, expected %s, err: %w", col.IndexFrom, col.Type.String(), ErrInvalidFieldType)
+	}
+
+	m[col.Name] = sval
+	return nil
+}

x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/group.go

@@ -0,0 +1,52 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package hostfs
+
+import (
+	"bufio"
+	"os"
+	"strings"
+)
+
+var columns = []ColumnInfo{
+	{0, "groupname", ColumnTypeString, false},
+	{2, "gid", ColumnTypeUint, false},
+	{2, "gid_signed", ColumnTypeInt, false},
+}
+
+func ReadGroup(fn string) ([]map[string]string, error) {
+	f, err := os.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var res []map[string]string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		fields := strings.Split(line, ":")
+
+		rec := make(StringMap)
+
+		for _, col := range columns {
+			err = rec.Set(fields, col)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		res = append(res, rec)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}

x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/passwd.go

@@ -0,0 +1,58 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package hostfs
+
+import (
+	"bufio"
+	"os"
+	"strings"
+)
+
+var passwdColumns = []ColumnInfo{
+	{0, "username", ColumnTypeString, false},
+	{2, "uid", ColumnTypeUint, false},
+	{2, "uid_signed", ColumnTypeInt, false},
+	{3, "gid", ColumnTypeUint, false},
+	{3, "gid_signed", ColumnTypeInt, false},
+	{4, "description", ColumnTypeString, false},
+	{5, "directory", ColumnTypeString, false},
+	{6, "shell", ColumnTypeString, false},
+	{7, "uuid", ColumnTypeString, true},
+}
+
+func ReadPasswd(fn string) ([]map[string]string, error) {
+	f, err := os.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var res []map[string]string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		fields := strings.Split(line, ":")
+
+		rec := make(StringMap)
+
+		for _, col := range passwdColumns {
+			err = rec.Set(fields, col)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		res = append(res, rec)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}

x-pack/osquerybeat/ext/osquery-extension/internal/proc/cmdline.go

@@ -0,0 +1,21 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package proc
+
+import (
+	"io/ioutil"
+	"strings"
+)
+
+func ReadCmdLine(root string, pid string) (string, error) {
+	fn := getProcAttr(root, pid, "cmdline")
+
+	b, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(b)), nil
+}

x-pack/osquerybeat/ext/osquery-extension/internal/proc/io.go

@@ -0,0 +1,57 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package proc
+
+import (
+	"bytes"
+	"io/ioutil"
+	"strings"
+)
+
+type ProcIO struct {
+	ReadBytes           string
+	WriteBytes          string
+	CancelledWriteBytes string
+}
+
+// ReadProcStat reads proccess stats from /proc/<pid>/io.
+// The parsing code logic is borrowed from osquery C++ implementation and translated to Go.
+// This makes the data returned from the `host_processes` table
+// consistent with data returned from the original osquery `processes` table.
+// https://github.com/osquery/osquery/blob/master/osquery/tables/system/linux/processes.cpp
+func ReadIO(root string, pid string) (procio ProcIO, err error) {
+	// Proc IO example
+	// rchar: 1527371144
+	// wchar: 1495591102
+	// syscr: 481186
+	// syscw: 255942
+	// read_bytes: 14401536
+	// write_bytes: 815329280
+	// cancelled_write_bytes: 40976384
+	fn := getProcAttr(root, pid, "io")
+	b, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return
+	}
+
+	lines := bytes.Split(b, []byte{'\n'})
+	for _, line := range lines {
+		detail := bytes.SplitN(line, []byte{':'}, 2)
+		if len(detail) != 2 {
+			continue
+		}
+
+		k := strings.TrimSpace(bytesToString(detail[0]))
+		switch k {
+		case "read_bytes":
+			procio.ReadBytes = strings.TrimSpace(bytesToString(detail[1]))
+		case "write_bytes":
+			procio.WriteBytes = strings.TrimSpace(bytesToString(detail[1]))
+		case "cancelled_write_bytes":
+			procio.CancelledWriteBytes = strings.TrimSpace(bytesToString(detail[1]))
+		}
+	}
+	return procio, nil
+}

x-pack/osquerybeat/ext/osquery-extension/internal/proc/link.go

@@ -0,0 +1,19 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package proc
+
+import (
+	"os"
+)
+
+func ReadLink(root string, pid string, attr string) (string, error) {
+	fn := getProcAttr(root, pid, attr)
+
+	s, err := os.Readlink(fn)
+	if err != nil {
+		return "", err
+	}
+	return s, nil
+}

x-pack/osquerybeat/ext/osquery-extension/internal/proc/list.go

@@ -0,0 +1,40 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package proc
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+func List(root string) ([]string, error) {
+	var pids []string
+
+	root = filepath.Join(root, "/proc")
+
+	dirs, err := os.ReadDir(root)
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, dir := range dirs {
+		if !dir.IsDir() {
+			continue
+		}
+
+		name := dir.Name()
+		// Check if directory is number
+		_, err := strconv.Atoi(name)
+		if err != nil {
+			err = nil
+			continue
+		}
+		pids = append(pids, name)
+	}
+
+	return pids, nil
+}