Skip to content

Commit

Permalink
Merge pull request #149 from coroot/fdinfo_optimization
Browse files Browse the repository at this point in the history 
file events: skip reading `fdinfo` for previously seen mounts
  • Loading branch information
@def
def authored Dec 4, 2024
2 parents 649b1bf + 1e16696 commit 98808a0
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 43 deletions.
19 changes: 16 additions & 3 deletions containers/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,8 @@ type Container struct {
oomKills int
pythonThreadLockWaitTime time.Duration

mounts map[string]proc.MountInfo
mounts map[string]proc.MountInfo
seenMounts map[uint64]struct{}

logParsers map[string]*LogParser

Expand Down Expand Up @@ -172,7 +173,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, pid
l7Stats: L7Stats{},
dnsStats: &L7Metrics{},

mounts: map[string]proc.MountInfo{},
mounts: map[string]proc.MountInfo{},
seenMounts: map[uint64]struct{}{},

logParsers: map[string]*LogParser{},

Expand Down Expand Up @@ -424,13 +426,24 @@ func (c *Container) onProcessExit(pid uint32, oomKill bool) {
}
}

func (c *Container) onFileOpen(pid uint32, fd uint64) {
func (c *Container) onFileOpen(pid uint32, fd uint64, mnt uint64, log bool) {
if mnt > 0 && !log {
c.lock.Lock()
_, ok := c.seenMounts[mnt]
c.lock.Unlock()
if ok {
return
}
}
mntId, logPath := resolveFd(pid, fd)
func() {
if mntId == "" {
return
}
c.lock.Lock()
if mnt > 0 {
c.seenMounts[mnt] = struct{}{}
}
_, ok := c.mounts[mntId]
c.lock.Unlock()
if ok {
Expand Down
2 changes: 1 addition & 1 deletion containers/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {

case ebpftracer.EventTypeFileOpen:
if c := r.getOrCreateContainer(e.Pid); c != nil {
c.onFileOpen(e.Pid, e.Fd)
c.onFileOpen(e.Pid, e.Fd, e.Mnt, e.Log)
}

case ebpftracer.EventTypeListenOpen:
Expand Down
4 changes: 4 additions & 0 deletions ebpftracer/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ WORKDIR /tmp/ebpf
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf416x86.o && llvm-strip --strip-debug ebpf416x86.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf420x86.o && llvm-strip --strip-debug ebpf420x86.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf506x86.o && llvm-strip --strip-debug ebpf506x86.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=507 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf507x86.o && llvm-strip --strip-debug ebpf507x86.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf512x86.o && llvm-strip --strip-debug ebpf512x86.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -D__CTX_EXTRA_PADDING -c ebpf.c -o ebpf512x86cep.o && llvm-strip --strip-debug ebpf512x86cep.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf416arm64.o && llvm-strip --strip-debug ebpf416arm64.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf420arm64.o && llvm-strip --strip-debug ebpf420arm64.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf506arm64.o && llvm-strip --strip-debug ebpf506arm64.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=507 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf507arm64.o && llvm-strip --strip-debug ebpf507arm64.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf512arm64.o && llvm-strip --strip-debug ebpf512arm64.o
RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_arm64 -D__CTX_EXTRA_PADDING -c ebpf.c -o ebpf512arm64cep.o && llvm-strip --strip-debug ebpf512arm64cep.o

Expand All @@ -25,13 +27,15 @@ RUN echo -en '// generated - do not edit\npackage ebpftracer\n\nvar ebpfProgs =
&& echo -en '\t\t{"5.12", "ctx-extra-padding", []byte("' >> ebpf.go && gzip -c ebpf512x86cep.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"5.12", "", []byte("' >> ebpf.go && gzip -c ebpf512x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"5.6", "", []byte("' >> ebpf.go && gzip -c ebpf506x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"5.7", "", []byte("' >> ebpf.go && gzip -c ebpf507x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"4.20", "", []byte("' >> ebpf.go && gzip -c ebpf420x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"4.16", "", []byte("' >> ebpf.go && gzip -c ebpf416x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t},\n'>> ebpf.go \
&& echo -en '\t"arm64": {\n' >> ebpf.go \
&& echo -en '\t\t{"5.12", "ctx-extra-padding", []byte("' >> ebpf.go && gzip -c ebpf512arm64cep.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"5.12", "", []byte("' >> ebpf.go && gzip -c ebpf512arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"5.6", "", []byte("' >> ebpf.go && gzip -c ebpf506arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"5.7", "", []byte("' >> ebpf.go && gzip -c ebpf507arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"4.20", "", []byte("' >> ebpf.go && gzip -c ebpf420arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t\t{"4.16", "", []byte("' >> ebpf.go && gzip -c ebpf416arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
&& echo -en '\t},\n'>> ebpf.go \
Expand Down
22 changes: 12 additions & 10 deletions ebpftracer/ebpf.go
View file

Large diffs are not rendered by default.

82 changes: 67 additions & 15 deletions ebpftracer/ebpf/file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ struct file_event {
__u32 type;
__u32 pid;
__u64 fd;
__u64 mnt;
__u64 log;
};

struct {
Expand All @@ -12,11 +14,24 @@ struct {
__uint(value_size, sizeof(int));
} file_events SEC(".maps");

struct path {
__u64 mnt;
};

struct nameidata {
struct path path;
};

struct file_info {
__u64 mnt;
__u64 log;
};

struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, sizeof(__u64));
__uint(value_size, sizeof(__u32));
__uint(max_entries, 10240);
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, sizeof(__u64));
__uint(value_size, sizeof(struct file_info));
__uint(max_entries, 10240);
} open_file_info SEC(".maps");

struct trace_event_raw_sys_enter_open__stub {
Expand All @@ -34,46 +49,83 @@ struct trace_event_raw_sys_enter_openat__stub {
long int flags;
};

static __always_inline
int do_open(struct pt_regs *ctx) {
__u64 id = bpf_get_current_pid_tgid();
struct nameidata nd;
if (bpf_probe_read_kernel(&nd, sizeof(nd), (void *)PT_REGS_PARM1(ctx)) != 0) {
return 0;
}
struct file_info *i = bpf_map_lookup_elem(&open_file_info, &id);
if (!i) {
return 0;
}
i->mnt = nd.path.mnt;
return 0;
}

#if __KERNEL_FROM >= 507
SEC("kprobe/do_open")
int do_open_kprobe(struct pt_regs *ctx) {
return do_open(ctx);
}
#else
SEC("kprobe/do_last")
int do_last_kprobe(struct pt_regs *ctx) {
return do_open(ctx);
}
#endif

static __always_inline
int trace_enter_open(long int flags, char *filename)
{
if (!(flags & O_ACCMODE & (O_WRONLY | O_RDWR))) {
return 0;
}
char p[7];
long res = bpf_probe_read_str(&p, sizeof(p), (void *)filename);
if (p[0]=='/' && p[1]=='p' && p[2]=='r' && p[3]=='o' && p[4]=='c' && p[5]=='/') {
char p[10];
bpf_probe_read_str(&p, sizeof(p), (void *)filename);
if (p[0] != '/') {
return 0;
}
if (p[1]=='p' && p[2]=='r' && p[3]=='o' && p[4]=='c' && p[5]=='/') {
return 0;
}
if (p[0]=='/' && p[1]=='d' && p[2]=='e' && p[3]=='v' && p[4]=='/') {
if (p[1]=='d' && p[2]=='e' && p[3]=='v' && p[4]=='/') {
return 0;
}
if (p[0]=='/' && p[1]=='s' && p[2]=='y' && p[3]=='s' && p[4]=='/') {
if (p[1]=='s' && p[2]=='y' && p[3]=='s' && p[4]=='/') {
return 0;
}
struct file_info i = {};
if (p[1]=='v' && p[2]=='a' && p[3]=='r' && p[4]=='/' && p[5]=='l' && p[6] == 'o' && p[7] == 'g' && p[8] == '/') {
i.log = 1;
}
__u64 id = bpf_get_current_pid_tgid();
__u32 v = 1;
bpf_map_update_elem(&open_file_info, &id, &v, BPF_ANY);
bpf_map_update_elem(&open_file_info, &id, &i, BPF_ANY);
return 0;
}

static __always_inline
int trace_exit_open(struct trace_event_raw_sys_exit__stub* ctx)
{
__u64 id = bpf_get_current_pid_tgid();
if (!bpf_map_lookup_elem(&open_file_info, &id)) {
return 0;
struct file_info *i = bpf_map_lookup_elem(&open_file_info, &id);
if (!i) {
return 0;
}
bpf_map_delete_elem(&open_file_info, &id);
if (ctx->ret < 0) {
if (ctx->ret < 0 || i->mnt == 0) {
bpf_map_delete_elem(&open_file_info, &id);
return 0;
}
struct file_event e = {
.type = EVENT_TYPE_FILE_OPEN,
.pid = id >> 32,
.fd = ctx->ret,
.mnt = i->mnt,
.log = i->log,
};
bpf_perf_event_output(ctx, &file_events, BPF_F_CURRENT_CPU, &e, sizeof(e));
bpf_map_delete_elem(&open_file_info, &id);
return 0;
}

Expand Down
15 changes: 2 additions & 13 deletions ebpftracer/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@ const (
IPProtoTCP uint8 = 6
)

type file struct {
pid uint32
fd uint64
}

type sock struct {
pid uint32
fd uint64
Expand Down Expand Up @@ -100,10 +95,7 @@ func (t *Tracer) init(ch chan<- Event) error {
return err
}
nss := map[string]map[string]sock{}
var (
files []file
socks []sock
)
var socks []sock

for _, pid := range pids {
ns, err := proc.GetNetNs(pid)
Expand Down Expand Up @@ -151,14 +143,11 @@ func (t *Tracer) init(ch chan<- Event) error {
socks = append(socks, s)
}
case strings.HasPrefix(fd.Dest, "/"):
files = append(files, file{pid: pid, fd: fd.Fd})
ch <- Event{Type: EventTypeFileOpen, Pid: pid, Fd: fd.Fd, Log: strings.HasPrefix(fd.Dest, "/var/log/")}
}
}
}

for _, fd := range files {
ch <- Event{Type: EventTypeFileOpen, Pid: fd.pid, Fd: fd.fd}
}
listens := map[uint64]bool{}
for _, s := range socks {
if s.Listen {
Expand Down
6 changes: 5 additions & 1 deletion ebpftracer/tracer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ type Event struct {
Duration time.Duration
L7Request *l7.RequestData
TrafficStats *TrafficStats
Mnt uint64
Log bool
}

type perfMapType uint8
Expand Down Expand Up @@ -328,6 +330,8 @@ type fileEvent struct {
Type EventType
Pid uint32
Fd uint64
Mnt uint64
Log uint64
}

type l7Event struct {
Expand Down Expand Up @@ -398,7 +402,7 @@ func runEventsReader(name string, r *perf.Reader, ch chan<- Event, typ perfMapTy
klog.Warningln("failed to read msg:", err)
continue
}
event = Event{Type: v.Type, Pid: v.Pid, Fd: v.Fd}
event = Event{Type: v.Type, Pid: v.Pid, Fd: v.Fd, Mnt: v.Mnt, Log: v.Log > 0}
case perfMapTypeProcEvents:
v := &procEvent{}
if err := binary.Read(bytes.NewBuffer(rec.RawSample), binary.LittleEndian, v); err != nil {
Expand Down

0 comments on commit 98808a0

Please sign in to comment.