From 3550969c7b66c71573ef77ca9eb59a7acdc443d6 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Wed, 5 Jan 2022 14:42:39 -0800 Subject: [PATCH] PoC for optional json encoding --- libbeat/opt/opt.go | 31 ++++++- .../internal/metrics/memory/marshalexp.go | 92 +++++++++++++++++++ metricbeat/internal/metrics/memory/memory.go | 41 ++++++++- .../internal/metrics/memory/memory_test.go | 89 ++++++++++++++++++ 4 files changed, 247 insertions(+), 6 deletions(-) create mode 100644 metricbeat/internal/metrics/memory/marshalexp.go diff --git a/libbeat/opt/opt.go b/libbeat/opt/opt.go index 37171e17272..0d332cfa60d 100644 --- a/libbeat/opt/opt.go +++ b/libbeat/opt/opt.go @@ -17,7 +17,16 @@ package opt -import "github.com/elastic/go-structform" +import ( + "strconv" + + "github.com/elastic/go-structform" +) + +type OptType interface { + IsZero() bool + MarshalJSON() ([]byte, error) +} // Uint @@ -28,6 +37,16 @@ type Uint struct { value uint64 } +// MarshalJSON implements the marshal interface +func (v Uint) MarshalJSON() ([]byte, error) { + //fmt.Printf("In custom marshaller for Uint: %#v\n", v) + if v.exists { + return []byte(strconv.Itoa(int(v.value))), nil + } else { + return []byte(strconv.Itoa(int(0))), nil + } +} + // NewUintNone returns a new OptUint wrapper func NewUintNone() Uint { return Uint{ @@ -93,6 +112,16 @@ type Float struct { value float64 } +// MarshalJSON implements the marshal interface +func (v Float) MarshalJSON() ([]byte, error) { + //fmt.Printf("In custom marshaller for Float: %#v\n", v) + if v.exists { + return []byte(strconv.FormatFloat(v.value, 'f', 6, 64)), nil + } else { + return []byte(strconv.FormatFloat(v.value, 'f', 6, 64)), nil + } +} + // NewFloatNone returns a new uint wrapper func NewFloatNone() Float { return Float{ diff --git a/metricbeat/internal/metrics/memory/marshalexp.go b/metricbeat/internal/metrics/memory/marshalexp.go new file mode 100644 index 00000000000..f22ea5ba238 --- /dev/null +++ b/metricbeat/internal/metrics/memory/marshalexp.go @@ -0,0 +1,92 @@ +package memory + +import ( + "encoding/json" + "fmt" + "reflect" + + "github.com/elastic/beats/v7/libbeat/opt" +) + +type CustomFloat interface { + IsZero() bool +} + +type ZeroTest struct { + Zero bool +} + +func (z ZeroTest) MarshalJSON() ([]byte, error) { + return json.Marshal(z.Zero) +} + +func (z ZeroTest) IsZero() bool { + return z.Zero +} + +type UsedMemStatsTest struct { + Raw float64 `json:"raw,omitempty"` + Iface opt.OptType `json:"iface,omitempty"` +} + +// func (s UsedMemStatsTest) MarshalJSON() ([]byte, error) { + +// type sAlias UsedMemStatsTest + +// if s.Iface.IsZero() { +// s.Iface = nil +// } + +// return json.Marshal(sAlias(s)) +// } + +type MarshalWrapper struct { + Butterfly interface{} +} + +func (m MarshalWrapper) MarshalJSON() ([]byte, error) { + + bv := reflect.ValueOf(m.Butterfly).Elem() + for i := 0; i < bv.NumField(); i++ { + if bv.Field(i).CanInterface() { + fiface := bv.Field(i).Interface() + zeroIface, ok := fiface.(CustomFloat) + if ok { + if zeroIface.IsZero() { + zeroField := reflect.ValueOf(m.Butterfly).Elem().Field(i) + fmt.Printf("===%v\n", zeroField.Type()) + if zeroField.CanSet() { + zeroField.Set(reflect.Zero(zeroField.Type())) + } else { + fmt.Printf("Can't Set field %v\n", zeroField.Type()) + } + } + + } + } + } + return json.Marshal(m.Butterfly) +} + +func runJsonMarshal(input UsedMemStatsTest) (string, error) { + + // testStat := UsedMemStats{ + // Pct: opt.FloatWith(2.3), + // Bytes: opt.UintWith(100), + // } + //zero := ZeroTest{Zero: false} + // testStat := UsedMemStatsTest{ + + // Raw: 1.0, + // Iface: opt.FloatWith(2), + // } + wrapper := MarshalWrapper{ + Butterfly: &input, + } + + val, err := json.MarshalIndent(&wrapper, " ", " ") + if err != nil { + return "", err + } + return string(val), nil +} diff --git a/metricbeat/internal/metrics/memory/memory.go b/metricbeat/internal/metrics/memory/memory.go index 79efbdd825e..7695ba79311 100644 --- a/metricbeat/internal/metrics/memory/memory.go +++ b/metricbeat/internal/metrics/memory/memory.go @@ -48,8 +48,8 @@ type Memory struct { // UsedMemStats wraps used.* memory metrics type UsedMemStats struct { - Pct opt.Float `struct:"pct,omitempty"` - Bytes opt.Uint `struct:"bytes,omitempty"` + Pct opt.Float `struct:"pct,omitempty" json:"pct,omitempty"` + Bytes opt.Uint `struct:"bytes,omitempty" json:"bytes,omitempty"` } // ActualMemoryMetrics wraps the actual.* memory metrics @@ -60,11 +60,42 @@ type ActualMemoryMetrics struct { // SwapMetrics wraps swap.* memory metrics type SwapMetrics struct { - Total opt.Uint `struct:"total,omitempty"` - Used UsedMemStats `struct:"used,omitempty"` - Free opt.Uint `struct:"free,omitempty"` + Total opt.Uint `struct:"total,omitempty" json:"total,omitempty"` + Used UsedMemStats `struct:"used,omitempty" json:"used,omitempty"` + Free opt.Uint `struct:"free,omitempty" json:"free,omitempty"` } +// func (m UsedMemStats) MarshalJSON() ([]byte, error) { + +// outMap := map[string]interface{}{} +// bv := reflect.ValueOf(m) +// bt := reflect.TypeOf(m) +// for i := 0; i < bv.NumField(); i++ { +// if bv.Field(i).CanInterface() { +// fiface := bv.Field(i).Interface() +// name := bt.Field(i).Name + +// zeroIface, ok := fiface.(opt.OptType) +// if ok { +// if zeroIface.IsZero() { +// continue +// } +// //fmt.Printf("marshalling type %#v\n", zeroIface) +// rawOut, err := json.Marshal(zeroIface) +// if err != nil { +// return nil, errors.Wrap(err, "error marshalling type from UsedMemStats") +// } +// outMap[name] = string(rawOut) + +// } else { +// outMap[name] = fiface +// } +// } +// } +// //fmt.Printf("Pre-marshal map is %#v\n", outMap) +// return json.Marshal(outMap) +// } + // Get returns platform-independent memory metrics. func Get(procfs resolve.Resolver) (Memory, error) { base, err := get(procfs) diff --git a/metricbeat/internal/metrics/memory/memory_test.go b/metricbeat/internal/metrics/memory/memory_test.go index 4c9c979fdce..535bbb6b98f 100644 --- a/metricbeat/internal/metrics/memory/memory_test.go +++ b/metricbeat/internal/metrics/memory/memory_test.go @@ -22,6 +22,8 @@ package memory import ( + "bytes" + "encoding/json" "runtime" "testing" @@ -29,8 +31,95 @@ import ( "github.com/elastic/beats/v7/libbeat/metric/system/resolve" "github.com/elastic/beats/v7/libbeat/opt" + "github.com/elastic/go-structform/gotype" + gsjson "github.com/elastic/go-structform/json" ) +func TestMarshal(t *testing.T) { + testStat := UsedMemStatsTest{ + Raw: 5, + Iface: opt.NewFloatNone(), + } + + jsonData, err := runJsonMarshal(testStat) + assert.NoError(t, err) + t.Logf("%s", jsonData) +} + +func TestStdLibJSON(t *testing.T) { + testStat := SwapMetrics{ + Total: opt.UintWith(5), + Free: opt.NewUintNone(), + Used: UsedMemStats{ + Pct: opt.FloatWith(4.5), + Bytes: opt.UintWith(5), + }, + } + out, err := json.Marshal(testStat) + assert.NoError(t, err, "Marshal") + t.Logf("Out: %s", string(out)) +} + +func TestStructform(t *testing.T) { + outBuf := new(bytes.Buffer) + visitor := gsjson.NewVisitor(outBuf) + folder, err := gotype.NewIterator(visitor, + gotype.Folders(), + ) + assert.NoError(t, err, "NewIterator") + err = runStructformEncoder(folder) + assert.NoError(t, err, "runStructformEncoder") + t.Logf("output from structform: %s", string(outBuf.Bytes())) +} + +func BenchmarkStdLibJSON(b *testing.B) { + testStat := UsedMemStatsTest{ + Raw: 5, + Iface: opt.FloatWith(4.3), + } + wrapper := MarshalWrapper{ + Butterfly: &testStat, + } + for i := 0; i < b.N; i++ { + json.Marshal(&wrapper) + } +} + +func BenchmarkStructform(b *testing.B) { + testStat := SwapMetrics{ + Total: opt.UintWith(5), + Free: opt.NewUintNone(), + Used: UsedMemStats{ + Pct: opt.FloatWith(4.5), + Bytes: opt.UintWith(5), + }, + } + outBuf := new(bytes.Buffer) + visitor := gsjson.NewVisitor(outBuf) + folder, err := gotype.NewIterator(visitor, + gotype.Folders(), + ) + if err != nil { + b.Fatalf("err: %s", err) + } + err = runStructformEncoder(folder) + if err != nil { + b.Fatalf("err: %s", err) + } + + for i := 0; i < b.N; i++ { + folder.Fold(testStat) + } +} + +func runStructformEncoder(folder *gotype.Iterator) error { + testStat := UsedMemStatsTest{ + Raw: 5, + Iface: opt.FloatWith(4.3), + } + return folder.Fold(testStat) +} + func TestGetMemory(t *testing.T) { mem, err := Get(resolve.NewTestResolver(""))