From ec8e0511b2c0d82282ca2d3127d11ac9650467b1 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 19 Oct 2021 22:18:38 -0400 Subject: [PATCH] Osquerybeat: Implement host_users, host_groups, host_processes tables as a part of our osquery_extension. (#28434) * Osquerybeat: Implement host_users, host_groups, host_processes tables as a part of our osquery_extension. * The expectation is that the /etc/passwd, /etc/group, /proc are avaiable in the container under /hostfs as: /hostfs/etc/passwd, /hostfs/etc/group, /hostfs/proc. The ELASTIC_OSQUERY_HOSTFS environment variable allows to change the default. * The on_disk column is always set to -1 * Remove erroneous comment line --- .../internal/hostfs/common.go | 92 ++++++++ .../internal/hostfs/group.go | 52 +++++ .../internal/hostfs/passwd.go | 58 +++++ .../internal/proc/cmdline.go | 21 ++ .../ext/osquery-extension/internal/proc/io.go | 57 +++++ .../osquery-extension/internal/proc/link.go | 19 ++ .../osquery-extension/internal/proc/list.go | 40 ++++ .../osquery-extension/internal/proc/stat.go | 120 ++++++++++ .../internal/proc/strconv.go | 11 + .../osquery-extension/internal/proc/uptime.go | 36 +++ .../internal/tables/host_groups.go | 32 +++ .../internal/tables/host_processes.go | 212 ++++++++++++++++++ .../internal/tables/host_users.go | 38 ++++ .../osquerybeat/ext/osquery-extension/main.go | 49 +++- .../ext/osquery-extension/main_darwin.go | 20 ++ .../ext/osquery-extension/main_linux.go | 21 ++ .../ext/osquery-extension/main_windows.go | 13 ++ x-pack/osquerybeat/magefile.go | 5 +- 18 files changed, 881 insertions(+), 15 deletions(-) create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/common.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/group.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/passwd.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/cmdline.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/io.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/link.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/list.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/stat.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/strconv.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/proc/uptime.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_groups.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_processes.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_users.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/main_darwin.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/main_linux.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/main_windows.go diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/common.go b/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/common.go new file mode 100644 index 00000000000..cbf28c75fea --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/common.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package hostfs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" +) + +const ( + defaultMount = "/hostfs" + envHostFSOverride = "ELASTIC_OSQUERY_HOSTFS" // Allows to override the mount point for hostfs, default is /hostfs +) + +var ( + ErrMissingField = errors.New("missing/invalid field") + ErrInvalidFieldType = errors.New("invalid field type") +) + +type ColumnType int + +const ( + ColumnTypeString ColumnType = iota + ColumnTypeInt + ColumnTypeUint +) + +func (c ColumnType) String() string { + return [...]string{"string", "int64", "uint64"}[c] +} + +type ColumnInfo struct { + IndexFrom int + Name string + Type ColumnType + Optional bool +} + +func GetPath(fp string) string { + // Check the environment variable for override, otherwise use /hostfs as the mount root + mountRoot := os.Getenv(envHostFSOverride) + if mountRoot == "" { + mountRoot = defaultMount + } + return filepath.Join(mountRoot, fp) +} + +type StringMap map[string]string + +func (m StringMap) Set(fields []string, col ColumnInfo) error { + if col.IndexFrom >= len(fields) { + if !col.Optional { + return fmt.Errorf("failed to read field at index: %d, when total number of fields is: %d, err: %w", col.IndexFrom, len(fields), ErrMissingField) + } + m[col.Name] = "" + return nil + } + + var err error + + sval := fields[col.IndexFrom] + // Check that it is convertable to int type + switch col.Type { + case ColumnTypeUint: + // For unsigned values (Apple) the number is parsed as signed int32 then converted to unsigned. + // This is consistent with osquery `users` table data on Mac OS. + // osquery> select * from users; + // +------------+------------+------------+------------+------------------------+-------------------------------------------------+-------------------------------+------------------+--------------------------------------+-----------+ + // | uid | gid | uid_signed | gid_signed | username | description | directory | shell | uuid | is_hidden | + // +------------+------------+------------+------------+------------------------+-------------------------------------------------+-------------------------------+------------------+--------------------------------------+-----------+ + // | 229 | 4294967294 | 229 | -2 | _avbdeviced | Ethernet AVB Device Daemon | /var/empty | /usr/bin/false | FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000E5 | 0 | + v, err := strconv.ParseInt(sval, 10, 32) + if err == nil { + n := uint32(v) + sval = strconv.FormatUint(uint64(n), 10) + } + case ColumnTypeInt: + _, err = strconv.ParseInt(sval, 10, 64) + } + + if err != nil { + return fmt.Errorf("invalid field type at index: %d, expected %s, err: %w", col.IndexFrom, col.Type.String(), ErrInvalidFieldType) + } + + m[col.Name] = sval + return nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/group.go b/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/group.go new file mode 100644 index 00000000000..9ece5e15a90 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/group.go @@ -0,0 +1,52 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package hostfs + +import ( + "bufio" + "os" + "strings" +) + +var columns = []ColumnInfo{ + {0, "groupname", ColumnTypeString, false}, + {2, "gid", ColumnTypeUint, false}, + {2, "gid_signed", ColumnTypeInt, false}, +} + +func ReadGroup(fn string) ([]map[string]string, error) { + f, err := os.Open(fn) + if err != nil { + return nil, err + } + defer f.Close() + + var res []map[string]string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "#") { + continue + } + fields := strings.Split(line, ":") + + rec := make(StringMap) + + for _, col := range columns { + err = rec.Set(fields, col) + if err != nil { + return nil, err + } + } + + res = append(res, rec) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return res, nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/passwd.go b/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/passwd.go new file mode 100644 index 00000000000..5a08f448a4e --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs/passwd.go @@ -0,0 +1,58 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package hostfs + +import ( + "bufio" + "os" + "strings" +) + +var passwdColumns = []ColumnInfo{ + {0, "username", ColumnTypeString, false}, + {2, "uid", ColumnTypeUint, false}, + {2, "uid_signed", ColumnTypeInt, false}, + {3, "gid", ColumnTypeUint, false}, + {3, "gid_signed", ColumnTypeInt, false}, + {4, "description", ColumnTypeString, false}, + {5, "directory", ColumnTypeString, false}, + {6, "shell", ColumnTypeString, false}, + {7, "uuid", ColumnTypeString, true}, +} + +func ReadPasswd(fn string) ([]map[string]string, error) { + f, err := os.Open(fn) + if err != nil { + return nil, err + } + defer f.Close() + + var res []map[string]string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "#") { + continue + } + fields := strings.Split(line, ":") + + rec := make(StringMap) + + for _, col := range passwdColumns { + err = rec.Set(fields, col) + if err != nil { + return nil, err + } + } + + res = append(res, rec) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return res, nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/cmdline.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/cmdline.go new file mode 100644 index 00000000000..3ed7c2bb047 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/cmdline.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import ( + "io/ioutil" + "strings" +) + +func ReadCmdLine(root string, pid string) (string, error) { + fn := getProcAttr(root, pid, "cmdline") + + b, err := ioutil.ReadFile(fn) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(b)), nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/io.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/io.go new file mode 100644 index 00000000000..e57af9b9eae --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/io.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import ( + "bytes" + "io/ioutil" + "strings" +) + +type ProcIO struct { + ReadBytes string + WriteBytes string + CancelledWriteBytes string +} + +// ReadProcStat reads proccess stats from /proc//io. +// The parsing code logic is borrowed from osquery C++ implementation and translated to Go. +// This makes the data returned from the `host_processes` table +// consistent with data returned from the original osquery `processes` table. +// https://github.com/osquery/osquery/blob/master/osquery/tables/system/linux/processes.cpp +func ReadIO(root string, pid string) (procio ProcIO, err error) { + // Proc IO example + // rchar: 1527371144 + // wchar: 1495591102 + // syscr: 481186 + // syscw: 255942 + // read_bytes: 14401536 + // write_bytes: 815329280 + // cancelled_write_bytes: 40976384 + fn := getProcAttr(root, pid, "io") + b, err := ioutil.ReadFile(fn) + if err != nil { + return + } + + lines := bytes.Split(b, []byte{'\n'}) + for _, line := range lines { + detail := bytes.SplitN(line, []byte{':'}, 2) + if len(detail) != 2 { + continue + } + + k := strings.TrimSpace(bytesToString(detail[0])) + switch k { + case "read_bytes": + procio.ReadBytes = strings.TrimSpace(bytesToString(detail[1])) + case "write_bytes": + procio.WriteBytes = strings.TrimSpace(bytesToString(detail[1])) + case "cancelled_write_bytes": + procio.CancelledWriteBytes = strings.TrimSpace(bytesToString(detail[1])) + } + } + return procio, nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/link.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/link.go new file mode 100644 index 00000000000..4725af9e1aa --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/link.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import ( + "os" +) + +func ReadLink(root string, pid string, attr string) (string, error) { + fn := getProcAttr(root, pid, attr) + + s, err := os.Readlink(fn) + if err != nil { + return "", err + } + return s, nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/list.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/list.go new file mode 100644 index 00000000000..be24adc292f --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/list.go @@ -0,0 +1,40 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import ( + "os" + "path/filepath" + "strconv" +) + +func List(root string) ([]string, error) { + var pids []string + + root = filepath.Join(root, "/proc") + + dirs, err := os.ReadDir(root) + + if err != nil { + return nil, err + } + + for _, dir := range dirs { + if !dir.IsDir() { + continue + } + + name := dir.Name() + // Check if directory is number + _, err := strconv.Atoi(name) + if err != nil { + err = nil + continue + } + pids = append(pids, name) + } + + return pids, nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/stat.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/stat.go new file mode 100644 index 00000000000..f36aa3461f0 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/stat.go @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import ( + "bytes" + "errors" + "io/ioutil" + "path/filepath" + "strings" +) + +var ( + ErrInvalidProcStatHeader = errors.New("invalid /proc/stat header") + ErrInvalidProcStatContent = errors.New("invalid /proc/stat content") +) + +type ProcStat struct { + Name string + RealUID string + RealGID string + EffectiveUID string + EffectiveGID string + SavedUID string + SavedGID string + ResidentSize string + TotalSize string + State string + Parent string + Group string + Nice string + Threads string + UserTime string + SystemTime string + StartTime string +} + +func getProcAttr(root, pid, attr string) string { + return filepath.Join(root, "/proc", pid, attr) +} + +// ReadProcStat reads proccess stats from /proc//stat. +// The parsing code logic is borrowed from osquery C++ implementation and translated to Go. +// This makes the data returned from the `host_processes` table +// consistent with data returned from the original osquery `processes` table. +// https://github.com/osquery/osquery/blob/master/osquery/tables/system/linux/processes.cpp +func ReadStat(root string, pid string) (stat ProcStat, err error) { + fn := getProcAttr(root, pid, "stat") + b, err := ioutil.ReadFile(fn) + if err != nil { + return + } + // Proc stat example + // 6462 (bash) S 6402 6462 6462 34817 37849 4194304 14126 901131 0 191 15 9 3401 725 20 0 1 0 134150 20156416 1369 18446744073709551615 94186936238080 94186936960773 140723699470016 0 0 0 65536 3670020 1266777851 1 0 0 17 7 0 0 0 0 0 94186937191664 94186937239044 94186967023616 140723699476902 140723699476912 140723699476912 140723699478510 0 + pos := bytes.IndexByte(b, ')') + if pos == -1 { + return stat, ErrInvalidProcStatHeader + } + + content := bytesToString(b[pos+2:]) + details := strings.Split(content, " ") + if len(details) < 19 { + return stat, ErrInvalidProcStatContent + } + + stat.State = details[0] + stat.Parent = details[1] + stat.Group = details[2] + stat.UserTime = details[11] + stat.SystemTime = details[12] + stat.Nice = details[16] + stat.Threads = details[17] + stat.StartTime = details[19] + + fn = getProcAttr(root, pid, "status") + b, err = ioutil.ReadFile(fn) + if err != nil { + return + } + + lines := bytes.Split(b, []byte{'\n'}) + for _, line := range lines { + detail := bytes.SplitN(line, []byte{':'}, 2) + if len(detail) != 2 { + continue + } + + k := strings.TrimSpace(bytesToString(detail[0])) + v := bytesToString(detail[1]) + switch k { + case "Name": + stat.Name = strings.TrimSpace(v) + case "VmRSS": + if len(v) >= 3 { + stat.ResidentSize = strings.TrimSpace(v[:len(v)-3] + "000") + } + case "VmSize": + if len(v) >= 3 { + stat.TotalSize = strings.TrimSpace(v[:len(v)-3] + "000") + } + case "Gid": + arr := strings.Split(v, "\t") + if len(arr) == 4 { + stat.RealGID = arr[0] + stat.EffectiveGID = arr[1] + stat.SavedGID = arr[2] + } + case "Uid": + arr := strings.Split(v, "\t") + if len(arr) == 4 { + stat.RealUID = arr[0] + stat.EffectiveUID = arr[1] + stat.SavedUID = arr[2] + } + } + } + return stat, err +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/strconv.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/strconv.go new file mode 100644 index 00000000000..320b7f5da22 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/strconv.go @@ -0,0 +1,11 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import "unsafe" + +func bytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/proc/uptime.go b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/uptime.go new file mode 100644 index 00000000000..1ede0601762 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/proc/uptime.go @@ -0,0 +1,36 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proc + +import ( + "bytes" + "errors" + "io/ioutil" + "path/filepath" + "strconv" +) + +var ( + ErrInvalidProcUptimecontent = errors.New("invalid /proc/uptime content") +) + +// Reads system uptime from /proc/uptime +func ReadUptime(root string) (secs int64, err error) { + fp := filepath.Join(root, "/proc/uptime") + b, err := ioutil.ReadFile(fp) + if err != nil { + return + } + detail := bytes.Split(b, []byte{' '}) + if len(detail) != 2 { + return secs, ErrInvalidProcUptimecontent + } + + num, err := strconv.ParseFloat(bytesToString(detail[0]), 64) + if err != nil { + return secs, ErrInvalidProcUptimecontent + } + return int64(num), nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_groups.go b/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_groups.go new file mode 100644 index 00000000000..1197f5753ef --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_groups.go @@ -0,0 +1,32 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package tables + +import ( + "context" + + "github.com/osquery/osquery-go/plugin/table" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs" +) + +const ( + groupFile = "/etc/group" +) + +func HostGroupsColumns() []table.ColumnDefinition { + return []table.ColumnDefinition{ + table.BigIntColumn("gid"), + table.BigIntColumn("gid_signed"), + table.TextColumn("groupname"), + } +} + +func GetHostGroupsGenerateFunc() table.GenerateFunc { + fn := hostfs.GetPath(groupFile) + return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + return hostfs.ReadGroup(fn) + } +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_processes.go b/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_processes.go new file mode 100644 index 00000000000..baf3f08f9e5 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_processes.go @@ -0,0 +1,212 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package tables + +import ( + "context" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/osquery/osquery-go/plugin/table" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs" + "github.com/elastic/beats/v7/x-pack/osquerybeat/ext/osquery-extension/internal/proc" +) + +const ( + procDir = "/proc" + + // https://groups.google.com/g/fa.linux.kernel/c/JndVy0RgHHI/m/Nu7nkRfZ-c0J + // CLK_TCK is 100 on x86. As it has always been. User land should never + // care about whatever random value the kernel happens to use for the + // actual timer tick at that particular moment. Especially since the kernel + // internal timer tick may well be variable some day. + + // The fact that libproc believes that HZ can change is _their_ problem. + // I've told people over and over that user-level HZ is a constant (and, on + // x86, that constant is 100), and that won't change. + + // So in current 2.5.x times() still counts at 100Hz, and /proc files that + // export clock_t still show the same 100Hz rate. + + // The fact that the kernel internally counts at some different rate should + // be _totally_ invisible to user programs (except they get better latency + // for stuff like select() and other timeouts). + + // Linus + clkTck = 100 + + msIn1CLKTCK = (1000 / clkTck) +) + +func HostProcessesColumns() []table.ColumnDefinition { + return []table.ColumnDefinition{ + table.BigIntColumn("pid"), + table.TextColumn("name"), + table.TextColumn("path"), + table.TextColumn("cmdline"), + table.TextColumn("state"), + table.TextColumn("cwd"), + table.TextColumn("root"), + table.BigIntColumn("uid"), + table.BigIntColumn("gid"), + table.BigIntColumn("euid"), + table.BigIntColumn("egid"), + table.BigIntColumn("suid"), + table.BigIntColumn("sgid"), + table.IntegerColumn("on_disk"), + table.BigIntColumn("wired_size"), + table.BigIntColumn("resident_size"), + table.BigIntColumn("total_size"), + table.BigIntColumn("user_time"), + table.BigIntColumn("system_time"), + table.BigIntColumn("disk_bytes_read"), + table.BigIntColumn("disk_bytes_written"), + table.BigIntColumn("start_time"), + table.BigIntColumn("parent"), + table.BigIntColumn("pgroup"), + table.IntegerColumn("threads"), + table.IntegerColumn("nice"), + } +} + +func GetHostProcessesGenerateFunc() table.GenerateFunc { + root := hostfs.GetPath("") + + return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + return genProcesses(root, queryContext) + } +} + +func genProcesses(root string, queryContext table.QueryContext) ([]map[string]string, error) { + systemBootTime, err := proc.ReadUptime(root) + if err != nil { + return nil, err + } + + if systemBootTime > 0 { + systemBootTime = time.Now().Unix() - systemBootTime + } + + pids, err := getProcList(root, queryContext) + if err != nil { + return nil, err + } + + var res []map[string]string + for _, pid := range pids { + rec := genProcess(root, pid, systemBootTime) + if rec != nil { + res = append(res, rec) + } + } + + return res, nil +} + +func dirExists(dirp string) (ok bool, err error) { + if stat, err := os.Stat(dirp); err == nil && stat.IsDir() { + ok = true + } else if os.IsNotExist(err) { + err = nil + } + return +} + +func getProcList(root string, queryContext table.QueryContext) ([]string, error) { + pidset := make(map[string]struct{}) + if contraintList, ok := queryContext.Constraints["pid"]; ok && len(contraintList.Constraints) > 0 { + for _, constraint := range contraintList.Constraints { + if constraint.Operator == table.OperatorEquals { + if ok, _ := dirExists(filepath.Join(root, constraint.Expression)); ok { + pidset[constraint.Expression] = struct{}{} + } + } + } + } + + // Enumerate all processes pids + if len(pidset) == 0 { + return proc.List(root) + } + + pids := make([]string, 0, len(pidset)) + for pid, _ := range pidset { + pids = append(pids, pid) + } + return pids, nil +} + +// genProcess can fail in multiple places, still return the full record. +// This is consistent with original osquery C++ implementation for processes records +func genProcess(root string, pid string, systemBootTime int64) map[string]string { + pstat, err := proc.ReadStat(root, pid) + if err != nil { + return nil + } + + r := make(map[string]string, 26) + + if procIO, err := proc.ReadIO(root, pid); err == nil { + r["disk_bytes_read"] = procIO.ReadBytes + written, _ := strconv.ParseUint(procIO.WriteBytes, 10, 64) + cancelled, _ := strconv.ParseUint(procIO.CancelledWriteBytes, 10, 64) + r["disk_bytes_written"] = strconv.FormatUint(written-cancelled, 10) + } + + r["pid"] = pid + r["parent"] = pstat.Parent + r["path"] = mustString(proc.ReadLink(root, pid, "exe")) + r["name"] = pstat.Name + r["pgroup"] = pstat.Group + r["state"] = pstat.State + r["nice"] = pstat.Nice + r["threads"] = pstat.Threads + + r["cmdline"] = mustString(proc.ReadCmdLine(root, pid)) + r["cwd"] = mustString(proc.ReadLink(root, pid, "cwd")) + r["root"] = mustString(proc.ReadLink(root, pid, "root")) + + r["uid"] = pstat.RealUID + r["euid"] = pstat.EffectiveUID + r["suid"] = pstat.SavedUID + + r["gid"] = pstat.RealGID + r["egid"] = pstat.EffectiveGID + r["sgid"] = pstat.SavedGID + + // Can't check if the file exists on the host machine, setting to -1 + r["on_disk"] = "-1" + + r["wired_size"] = "0" + r["resident_size"] = pstat.ResidentSize + r["total_size"] = pstat.TotalSize + + r["user_time"] = formatClicks(pstat.UserTime) + r["system_time"] = formatClicks(pstat.SystemTime) + + pst, err := strconv.ParseInt(pstat.StartTime, 10, 64) + if err != nil || systemBootTime == 0 { + r["start_time"] = "-1" + } else { + r["start_time"] = strconv.FormatInt(systemBootTime+pst/clkTck, 10) + } + + return r +} + +func mustString(s string, err error) string { + return s +} + +func formatClicks(clicks string) string { + n, err := strconv.ParseUint(clicks, 10, 64) + if err != nil { + return "" + } + return strconv.FormatUint(n*msIn1CLKTCK, 10) +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_users.go b/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_users.go new file mode 100644 index 00000000000..52a3b084745 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/internal/tables/host_users.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package tables + +import ( + "context" + + "github.com/osquery/osquery-go/plugin/table" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/ext/osquery-extension/internal/hostfs" +) + +const ( + passwdFile = "/etc/passwd" +) + +func HostUsersColumns() []table.ColumnDefinition { + return []table.ColumnDefinition{ + table.BigIntColumn("uid"), + table.BigIntColumn("gid"), + table.BigIntColumn("uid_signed"), + table.BigIntColumn("gid_signed"), + table.TextColumn("username"), + table.TextColumn("description"), + table.TextColumn("directory"), + table.TextColumn("shell"), + table.TextColumn("uuid"), + } +} + +func GetHostUsersGenerateFunc() table.GenerateFunc { + fn := hostfs.GetPath(passwdFile) + return func(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + return hostfs.ReadPasswd(fn) + } +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/main.go b/x-pack/osquerybeat/ext/osquery-extension/main.go index 659e355bcaf..1e1f9923e7c 100644 --- a/x-pack/osquerybeat/ext/osquery-extension/main.go +++ b/x-pack/osquerybeat/ext/osquery-extension/main.go @@ -27,32 +27,59 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +// Expanded with Elastic custom extensions so we have only one binary to manager + package main import ( "flag" "fmt" + "log" "os" - "os/signal" "time" + + "github.com/osquery/osquery-go" +) + +var ( + socket = flag.String("socket", "", "Path to the extensions UNIX domain socket") + timeout = flag.Int("timeout", 3, "Seconds to wait for autoloaded extensions") + interval = flag.Int("interval", 3, "Seconds delay between connectivity checks") + verbose = flag.Bool("verbose", false, "Verbose logging") ) func main() { - var ( - _ = flag.Bool("verbose", false, "") - _ = flag.Int("interval", 0, "") - _ = flag.Int("timeout", 0, "") - _ = flag.String("socket", "", "") - ) flag.Parse() - fmt.Fprintf(os.Stderr, "%+v", os.Args) + if *socket == "" { + log.Fatalln("Missing required --socket argument") + } + + serverTimeout := osquery.ServerTimeout( + time.Second * time.Duration(*timeout), + ) + serverPingInterval := osquery.ServerPingInterval( + time.Second * time.Duration(*interval), + ) go monitorForParent() - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - <-sig + server, err := osquery.NewExtensionManagerServer( + "osquery-extension", + *socket, + serverTimeout, + serverPingInterval, + ) + if err != nil { + log.Fatalf("Error creating extension: %s\n", err) + } + + // Register the tables avaiable for the specific pltaform build + RegisterTables(server) + + if err := server.Run(); err != nil { + log.Fatal(err) + } } // continuously monitor for ppid and exit if osqueryd is no longer the parent process. diff --git a/x-pack/osquerybeat/ext/osquery-extension/main_darwin.go b/x-pack/osquerybeat/ext/osquery-extension/main_darwin.go new file mode 100644 index 00000000000..eb9fb591e4c --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/main_darwin.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build darwin +// +build darwin + +package main + +import ( + "github.com/osquery/osquery-go" + "github.com/osquery/osquery-go/plugin/table" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/ext/osquery-extension/internal/tables" +) + +func RegisterTables(server *osquery.ExtensionManagerServer) { + server.RegisterPlugin(table.NewPlugin("host_users", tables.HostUsersColumns(), tables.GetHostUsersGenerateFunc())) + server.RegisterPlugin(table.NewPlugin("host_groups", tables.HostGroupsColumns(), tables.GetHostGroupsGenerateFunc())) +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/main_linux.go b/x-pack/osquerybeat/ext/osquery-extension/main_linux.go new file mode 100644 index 00000000000..953e35974ba --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/main_linux.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux +// +build linux + +package main + +import ( + "github.com/osquery/osquery-go" + "github.com/osquery/osquery-go/plugin/table" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/ext/osquery-extension/internal/tables" +) + +func RegisterTables(server *osquery.ExtensionManagerServer) { + server.RegisterPlugin(table.NewPlugin("host_users", tables.HostUsersColumns(), tables.GetHostUsersGenerateFunc())) + server.RegisterPlugin(table.NewPlugin("host_groups", tables.HostGroupsColumns(), tables.GetHostGroupsGenerateFunc())) + server.RegisterPlugin(table.NewPlugin("host_processes", tables.HostProcessesColumns(), tables.GetHostProcessesGenerateFunc())) +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/main_windows.go b/x-pack/osquerybeat/ext/osquery-extension/main_windows.go new file mode 100644 index 00000000000..bc541b8b79e --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/main_windows.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows +// +build windows + +package main + +import "github.com/osquery/osquery-go" + +func RegisterTables(server *osquery.ExtensionManagerServer) { +} diff --git a/x-pack/osquerybeat/magefile.go b/x-pack/osquerybeat/magefile.go index d3e1923561b..2d4e9a1b7db 100644 --- a/x-pack/osquerybeat/magefile.go +++ b/x-pack/osquerybeat/magefile.go @@ -47,12 +47,9 @@ func Build() error { return err } - // Building osquery-extension.ext - inputFiles := filepath.Join("ext/osquery-extension/main.go") - params.InputFiles = []string{inputFiles} + params.InputFiles = []string{"./ext/osquery-extension/."} params.Name = "osquery-extension" params.CGO = false - params.Env = make(map[string]string) err = devtools.Build(params) if err != nil { return err