From b0de04d04a6c1583b4989f6ca1c2e91675e386ca Mon Sep 17 00:00:00 2001 From: iamryanchia Date: Mon, 26 Aug 2024 12:12:56 +0800 Subject: [PATCH 1/4] feat: add jsonextracter package --- go.mod | 9 + go.sum | 24 ++ jsonextracter/README.md | 107 ++++++ jsonextracter/alias.go | 40 ++ jsonextracter/extracter.go | 59 +++ jsonextracter/extracter_test.go | 57 +++ jsonextracter/fieldpath.go | 71 ++++ jsonextracter/fieldpath_test.go | 90 +++++ jsonextracter/jsonpath.go | 655 ++++++++++++++++++++++++++++++++ jsonextracter/jsonpath_test.go | 171 +++++++++ jsonextracter/merge.go | 67 ++++ jsonextracter/merge_test.go | 133 +++++++ 12 files changed, 1483 insertions(+) create mode 100644 jsonextracter/README.md create mode 100644 jsonextracter/alias.go create mode 100644 jsonextracter/extracter.go create mode 100644 jsonextracter/extracter_test.go create mode 100644 jsonextracter/fieldpath.go create mode 100644 jsonextracter/fieldpath_test.go create mode 100644 jsonextracter/jsonpath.go create mode 100644 jsonextracter/jsonpath_test.go create mode 100644 jsonextracter/merge.go create mode 100644 jsonextracter/merge_test.go diff --git a/go.mod b/go.mod index 5411477..9dd5b37 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module kusionstack.io/kube-utils go 1.19 require ( + github.com/Masterminds/sprig/v3 v3.2.3 github.com/go-logr/logr v1.4.1 github.com/hashicorp/consul/sdk v0.16.0 github.com/onsi/ginkgo v1.16.5 @@ -23,6 +24,8 @@ require ( ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -36,9 +39,12 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect + github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nxadm/tail v1.4.8 // indirect @@ -47,9 +53,12 @@ require ( github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.16.0 // indirect diff --git a/go.sum b/go.sum index 3c7eeda..ed08588 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,12 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -237,8 +243,11 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -288,6 +297,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -296,6 +307,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -376,6 +389,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -392,6 +407,8 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg= github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -478,8 +495,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -537,6 +557,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -611,6 +632,7 @@ golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -621,6 +643,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= @@ -635,6 +658,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= diff --git a/jsonextracter/README.md b/jsonextracter/README.md new file mode 100644 index 0000000..417dc65 --- /dev/null +++ b/jsonextracter/README.md @@ -0,0 +1,107 @@ +# JSON Extracter + +Extract specific field from JSON and **output not only the field value but also its upstream structure**. + +A typical use case is to trim k8s objects in `TransformingInformer` to save informer memory. + +Please refer to [JSONPath Support](https://kubernetes.io/docs/reference/kubectl/jsonpath/) to see JSONPath usage. + +## Example + +Code: + +```go +package main + +import ( + "encoding/json" + "fmt" + + "kusionstack.io/kube-utils/jsonextracter" +) + +var pod = []byte(`{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "labels": { + "name": "pause", + "app": "pause" + }, + "name": "pause", + "namespace": "default" + }, + "spec": { + "containers": [ + { + "image": "registry.k8s.io/pause:3.8", + "imagePullPolicy": "IfNotPresent", + "name": "pause1" + }, + { + "image": "registry.k8s.io/pause:3.8", + "imagePullPolicy": "IfNotPresent", + "name": "pause2" + } + ] + } +}`) + +func printJSON(data interface{}) { + bytes, _ := json.Marshal(data) + fmt.Println(string(bytes)) +} + +func main() { + var podData map[string]interface{} + json.Unmarshal(pod, &podData) + + kindPath := "{.kind}" + kindExtracter, _ := jsonextracter.BuildExtracter(kindPath, false) + + kind, _ := kindExtracter.Extract(podData) + printJSON(kind) + + nameImagePath := "{.spec.containers[*]['name', 'image']}" + nameImageExtracter, _ := jsonextracter.BuildExtracter(nameImagePath, false) + + nameImage, _ := nameImageExtracter.Extract(podData) + printJSON(nameImage) + + merged, _ := jsonextracter.Merge([]jsonextracter.Extracter{kindExtracter, nameImageExtracter}, podData) + printJSON(merged) +} +``` + +Output: + +```plain +{"kind":"Pod"} +{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","name":"pause1"},{"image":"registry.k8s.io/pause:3.8","name":"pause2"}]}} +{"kind":"Pod","spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","name":"pause1"},{"image":"registry.k8s.io/pause:3.8","name":"pause2"}]}} +``` + +## Note + +The merge behavior of the `jsonextracter.Merge` on the list is replacing. Therefore, if you retrieve the container name and image separately and merge them, the resulting output will not contain the image. + +Code: + +```go + ... + namePath := "{.spec.containers[*].name}" + nameExtracter, _ := jsonextracter.BuildExtracter(namePath, false) + + imagePath := "{.spec.containers[*].image}" + imageExtracter, _ := jsonextracter.BuildExtracter(imagePath, false) + + merged, _ = jsonextracter.Merge([]jsonextracter.Extracter{imageExtracter, nameExtracter}, podData) + printJSON(merged) + ... +``` + +Output: + +```plain +{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}} +``` diff --git a/jsonextracter/alias.go b/jsonextracter/alias.go new file mode 100644 index 0000000..96e3df2 --- /dev/null +++ b/jsonextracter/alias.go @@ -0,0 +1,40 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "k8s.io/client-go/util/jsonpath" +) + +type ( + Parser = jsonpath.Parser + Node = jsonpath.Node + ListNode = jsonpath.ListNode + TextNode = jsonpath.TextNode + FieldNode = jsonpath.FieldNode + ArrayNode = jsonpath.ArrayNode + FilterNode = jsonpath.FilterNode + IntNode = jsonpath.IntNode + BoolNode = jsonpath.BoolNode + FloatNode = jsonpath.FloatNode + WildcardNode = jsonpath.WildcardNode + RecursiveNode = jsonpath.RecursiveNode + UnionNode = jsonpath.UnionNode + IdentifierNode = jsonpath.IdentifierNode +) + +var Parse = jsonpath.Parse diff --git a/jsonextracter/extracter.go b/jsonextracter/extracter.go new file mode 100644 index 0000000..ba5b480 --- /dev/null +++ b/jsonextracter/extracter.go @@ -0,0 +1,59 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "fmt" + + "k8s.io/client-go/util/jsonpath" +) + +type Extracter interface { + Extract(data map[string]interface{}) (map[string]interface{}, error) +} + +// BuildExtracter automatically determines whether to use FieldPathExtracter or JSONPathExtracter. +// If the input jsonPath only involves map operations, it will return FieldPathExtracter, +// as it has better performance. +func BuildExtracter(jsonPath string, allowMissingKeys bool) (Extracter, error) { + parser, err := Parse(jsonPath, jsonPath) + if err != nil { + return nil, fmt.Errorf("error in parsing path %q: %w", jsonPath, err) + } + + rootNodes := parser.Root.Nodes + if len(rootNodes) == 0 { + return NewNestedFieldPath(nil, allowMissingKeys), nil + } + + if len(rootNodes) == 1 { + nodes := rootNodes[0].(*jsonpath.ListNode).Nodes + fields := make([]string, 0, len(nodes)) + for _, node := range nodes { + if node.Type() == jsonpath.NodeField { + fields = append(fields, node.(*jsonpath.FieldNode).Value) + } + } + + if len(nodes) == len(fields) { + return NewNestedFieldPath(fields, allowMissingKeys), nil + } + } + + jp := &JSONPath{name: parser.Name, parser: parser, allowMissingKeys: allowMissingKeys} + return jp, nil +} diff --git a/jsonextracter/extracter_test.go b/jsonextracter/extracter_test.go new file mode 100644 index 0000000..70d9d21 --- /dev/null +++ b/jsonextracter/extracter_test.go @@ -0,0 +1,57 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "reflect" + "testing" +) + +func TestBuildExtracter(t *testing.T) { + type args struct { + path string + allowMissingKeys bool + } + tests := []struct { + name string + args args + want Extracter + wantErr bool + }{ + {name: "invalid path", args: args{path: `{`, allowMissingKeys: false}, want: nil, wantErr: true}, + {name: "fieldPath extracter", args: args{path: `{}`, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, + {name: "fieldPath extracter", args: args{path: ``, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, + {name: "fieldPath extracter", args: args{path: `{.metadata.labels.name}`, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, + {name: "fieldPath extracter", args: args{path: `{.metadata.labels['name']}`, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, + {name: "jsonPath extracter", args: args{path: `{.metadata.labels.name}{.metadata.labels.app}`, allowMissingKeys: false}, want: &JSONPath{}, wantErr: false}, + {name: "jsonPath extracter", args: args{path: `{.metadata.labels['name', 'app']}`, allowMissingKeys: false}, want: &JSONPath{}, wantErr: false}, + {name: "jsonPath extracter", args: args{path: `{.spec.containers[*].name}`, allowMissingKeys: false}, want: &JSONPath{}, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildExtracter(tt.args.path, tt.args.allowMissingKeys) + if (err != nil) != tt.wantErr { + t.Errorf("BuildExtracter() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if reflect.TypeOf(tt.want) != reflect.TypeOf(got) { + t.Errorf("BuildExtracter() = %T, want %T", got, tt.want) + } + }) + } +} diff --git a/jsonextracter/fieldpath.go b/jsonextracter/fieldpath.go new file mode 100644 index 0000000..3bea37b --- /dev/null +++ b/jsonextracter/fieldpath.go @@ -0,0 +1,71 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "fmt" +) + +// NewNestedFieldPath constructs a FieldPathExtracter. +func NewNestedFieldPath(nestedField []string, allowMissingKeys bool) *NestedFieldPath { + return &NestedFieldPath{nestedField: nestedField, allowMissingKeys: allowMissingKeys} +} + +// NestedFieldPath is used to wrap NestedFieldNoCopy function as an Extracter. +type NestedFieldPath struct { + nestedField []string + allowMissingKeys bool +} + +// Extract outputs the nestedField's value and its upstream structure. +func (f *NestedFieldPath) Extract(data map[string]interface{}) (map[string]interface{}, error) { + return NestedFieldNoCopy(data, f.allowMissingKeys, f.nestedField...) +} + +// NestedFieldNoCopy is similar to JSONPath.Extract. The difference is that it +// can only operate on map and does not support list, but has better performance. +func NestedFieldNoCopy(data map[string]interface{}, allowMissingKeys bool, fields ...string) (map[string]interface{}, error) { + if len(fields) == 0 { + return nil, nil + } + + result := map[string]interface{}{} + cur := result + + for i, field := range fields { + if val, ok := data[field]; ok { + if i != len(fields)-1 { + if data, ok = val.(map[string]interface{}); !ok { + return nil, fmt.Errorf("%v is of the type %T, expected map[string]interface{}", val, val) + } + + m := map[string]interface{}{} + cur[field] = m + cur = m + } else { + cur[field] = val + } + } else { + if allowMissingKeys { + return result, nil + } + return nil, fmt.Errorf("field %q not exist", field) + } + } + + return result, nil +} diff --git a/jsonextracter/fieldpath_test.go b/jsonextracter/fieldpath_test.go new file mode 100644 index 0000000..9e03b7c --- /dev/null +++ b/jsonextracter/fieldpath_test.go @@ -0,0 +1,90 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func TestFieldPath(t *testing.T) { + type args struct { + obj map[string]interface{} + allowMissingKeys bool + fields []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"empty", args{obj: podData, allowMissingKeys: true, fields: []string{}}, `null`, false}, + {"nil fields", args{obj: podData, allowMissingKeys: true, fields: nil}, `null`, false}, + {"nil input nil fields", args{obj: nil, allowMissingKeys: true, fields: nil}, `null`, false}, + {"nil input non-nil fields", args{obj: nil, allowMissingKeys: true, fields: []string{"xx"}}, `{}`, false}, + {"nil input non-nil fields not allow missing", args{obj: nil, allowMissingKeys: false, fields: []string{"xx"}}, `null`, true}, + + {"kind", args{obj: podData, allowMissingKeys: true, fields: []string{"kind"}}, `{"kind":"Pod"}`, false}, + {"lables", args{obj: podData, allowMissingKeys: true, fields: []string{"metadata", "labels"}}, `{"metadata":{"labels":{"app":"pause","name":"pause"}}}`, false}, + {"label name", args{obj: podData, allowMissingKeys: true, fields: []string{"metadata", "labels", "name"}}, `{"metadata":{"labels":{"name":"pause"}}}`, false}, + {"containers", args{obj: podData, allowMissingKeys: true, fields: []string{"spec", "containers"}}, `{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","imagePullPolicy":"IfNotPresent","name":"pause1","resources":{"limits":{"cpu":"100m","memory":"128Mi"},"requests":{"cpu":"100m","memory":"128Mi"}}},{"image":"registry.k8s.io/pause:3.8","imagePullPolicy":"IfNotPresent","name":"pause2","resources":{"limits":{"cpu":"10m","memory":"64Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}}]}}`, false}, + {"wrong type", args{obj: podData, allowMissingKeys: true, fields: []string{"metadata", "labels", "name", "xx"}}, "null", true}, + {"not allow miss key", args{obj: podData, allowMissingKeys: false, fields: []string{"metadata", "labels", "xx"}}, "null", true}, + {"allow miss key", args{obj: podData, allowMissingKeys: true, fields: []string{"metadata", "labels", "xx"}}, `{"metadata":{"labels":{}}}`, false}, + {"arbitrary", args{obj: arbitrary, allowMissingKeys: true, fields: []string{"e"}}, `{"e":{"f1":"f1","f2":"f2"}}`, false}, + {"not map", args{obj: arbitrary, allowMissingKeys: true, fields: []string{"e", "f1"}}, `null`, true}, + } + + fieldPathToJSONPath := func(nestedField []string) string { + if nestedField == nil { + return "" + } + if len(nestedField) == 0 { + return "{}" + } + + return fmt.Sprintf("{.%s}", strings.Join(nestedField, ".")) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NestedFieldNoCopy(tt.args.obj, tt.args.allowMissingKeys, tt.args.fields...) + + jpt := jsonPathTest{tt.name, fieldPathToJSONPath(tt.args.fields), tt.args.obj, tt.want, tt.wantErr} + testJSONPath([]jsonPathTest{jpt}, tt.args.allowMissingKeys, t) + + if (err != nil) != tt.wantErr { + t.Errorf("NestedFieldNoCopy() error = %v, wantErr %v", err, tt.wantErr) + return + } + + data, _ := json.Marshal(got) + if string(data) != tt.want { + t.Errorf("NestedFieldNoCopy() = %v, want %v", string(data), tt.want) + } + }) + } +} + +func BenchmarkFieldPath(b *testing.B) { + for n := 0; n < b.N; n++ { + NestedFieldNoCopy(podData, false, "kind") + } +} diff --git a/jsonextracter/jsonpath.go b/jsonextracter/jsonpath.go new file mode 100644 index 0000000..4c4620e --- /dev/null +++ b/jsonextracter/jsonpath.go @@ -0,0 +1,655 @@ +/** + * Copyright 2015 The Kubernetes Authors. + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Copied and adapted from https://github.com/kubernetes/client-go/blob/master/util/jsonpath/jsonpath.go + +package jsonextracter + +import ( + "fmt" + "reflect" + "strings" + + "k8s.io/client-go/third_party/forked/golang/template" +) + +type JSONPath struct { + name string + parser *Parser + beginRange int + inRange int + endRange int + + lastEndNode *Node + + allowMissingKeys bool +} + +// New creates a new JSONPath with the given name. +func New(name string) *JSONPath { + return &JSONPath{ + name: name, + beginRange: 0, + inRange: 0, + endRange: 0, + } +} + +// AllowMissingKeys allows a caller to specify whether they want an error if a field or map key +// cannot be located, or simply an empty result. The receiver is returned for chaining. +func (j *JSONPath) AllowMissingKeys(allow bool) *JSONPath { + j.allowMissingKeys = allow + return j +} + +// Parse parses the given template and returns an error. +func (j *JSONPath) Parse(text string) error { + var err error + j.parser, err = Parse(j.name, text) + return err +} + +type setFieldFunc func(val reflect.Value) error + +var nopSetFieldFunc = func(_ reflect.Value) error { return nil } + +func makeNopSetFieldFuncSlice(n int) []setFieldFunc { + fns := make([]setFieldFunc, n) + for i := 0; i < n; i++ { + fns[i] = nopSetFieldFunc + } + return fns +} + +// Extract outputs the field specified by JSONPath. +// The output contains not only the field value, but also its upstream structure. +// +// The data structure of the extracted field must be of type `map[string]interface{}`, +// and `struct` is not supported (an error will be returned). +func (j *JSONPath) Extract(data map[string]interface{}) (map[string]interface{}, error) { + container := struct{ Root reflect.Value }{} + setFn := func(val reflect.Value) error { + container.Root = val + return nil + } + + _, err := j.FindResults(data, setFn) + if err != nil { + return nil, err + } + + if !container.Root.IsValid() { + return nil, nil + } + + return container.Root.Interface().(map[string]interface{}), nil +} + +func (j *JSONPath) FindResults(data interface{}, setFn setFieldFunc) ([][]reflect.Value, error) { + if j.parser == nil { + return nil, fmt.Errorf("%s is an incomplete jsonpath template", j.name) + } + + cur := []reflect.Value{reflect.ValueOf(data)} + curnFn := []setFieldFunc{setFn} + nodes := j.parser.Root.Nodes + fullResult := [][]reflect.Value{} + for i := 0; i < len(nodes); i++ { + node := nodes[i] + results, fn, err := j._walk(cur, node, curnFn) + if err != nil { + return nil, err + } + + // encounter an end node, break the current block + if j.endRange > 0 && j.endRange <= j.inRange { + j.endRange-- + j.lastEndNode = &nodes[i] + break + } + // encounter a range node, start a range loop + if j.beginRange > 0 { + j.beginRange-- + j.inRange++ + if len(results) > 0 { + for ri, value := range results { + j.parser.Root.Nodes = nodes[i+1:] + nextResults, err := j.FindResults(value.Interface(), fn[ri]) + if err != nil { + return nil, err + } + fullResult = append(fullResult, nextResults...) + } + } else { + // If the range has no results, we still need to process the nodes within the range + // so the position will advance to the end node + j.parser.Root.Nodes = nodes[i+1:] + _, err := j.FindResults(nil, nopSetFieldFunc) + if err != nil { + return nil, err + } + } + j.inRange-- + + // Fast forward to resume processing after the most recent end node that was encountered + for k := i + 1; k < len(nodes); k++ { + if &nodes[k] == j.lastEndNode { + i = k + break + } + } + continue + } + fullResult = append(fullResult, results) + } + return fullResult, nil +} + +func (j *JSONPath) _walk(value []reflect.Value, node Node, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + switch node := node.(type) { + case *ListNode: + return j._evalList(value, node, setFn) + case *FieldNode: + return j.evalField(value, node, setFn) + case *ArrayNode: + return j.evalArray(value, node, setFn) + case *IdentifierNode: + return j.evalIdentifier(value, node, setFn) + case *UnionNode: + return j._evalUnion(value, node, setFn) + case *FilterNode: + return j.evalFilter(value, node, setFn) + default: + return nil, nil, fmt.Errorf("Extract does not support node %v", node) + } +} + +// walk visits tree rooted at the given node in DFS order +func (j *JSONPath) walk(value []reflect.Value, node Node) ([]reflect.Value, error) { + switch node := node.(type) { + case *ListNode: + return j.evalList(value, node) + case *TextNode: + return []reflect.Value{reflect.ValueOf(node.Text)}, nil + case *FieldNode: + value, _, err := j.evalField(value, node, makeNopSetFieldFuncSlice(len(value))) + return value, err + case *ArrayNode: + value, _, err := j.evalArray(value, node, makeNopSetFieldFuncSlice(len(value))) + return value, err + case *FilterNode: + value, _, err := j.evalFilter(value, node, makeNopSetFieldFuncSlice(len(value))) + return value, err + case *IntNode: + return j.evalInt(value, node) + case *BoolNode: + return j.evalBool(value, node) + case *FloatNode: + return j.evalFloat(value, node) + case *WildcardNode: + return j.evalWildcard(value, node) + case *RecursiveNode: + return j.evalRecursive(value, node) + case *UnionNode: + return j.evalUnion(value, node) + case *IdentifierNode: + value, _, err := j.evalIdentifier(value, node, makeNopSetFieldFuncSlice(len(value))) + return value, err + default: + return value, fmt.Errorf("unexpected Node %v", node) + } +} + +// evalInt evaluates IntNode +func (j *JSONPath) evalInt(input []reflect.Value, node *IntNode) ([]reflect.Value, error) { + result := make([]reflect.Value, len(input)) + for i := range input { + result[i] = reflect.ValueOf(node.Value) + } + return result, nil +} + +// evalFloat evaluates FloatNode +func (j *JSONPath) evalFloat(input []reflect.Value, node *FloatNode) ([]reflect.Value, error) { + result := make([]reflect.Value, len(input)) + for i := range input { + result[i] = reflect.ValueOf(node.Value) + } + return result, nil +} + +// evalBool evaluates BoolNode +func (j *JSONPath) evalBool(input []reflect.Value, node *BoolNode) ([]reflect.Value, error) { + result := make([]reflect.Value, len(input)) + for i := range input { + result[i] = reflect.ValueOf(node.Value) + } + return result, nil +} + +func (j *JSONPath) _evalList(value []reflect.Value, node *ListNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + var err error + curValue := value + curFns := setFn + + for _, node := range node.Nodes { + curValue, curFns, err = j._walk(curValue, node, curFns) + if err != nil { + return curValue, curFns, err + } + } + return curValue, curFns, nil +} + +// evalList evaluates ListNode +func (j *JSONPath) evalList(value []reflect.Value, node *ListNode) ([]reflect.Value, error) { + var err error + curValue := value + for _, node := range node.Nodes { + curValue, err = j.walk(curValue, node) + if err != nil { + return curValue, err + } + } + return curValue, nil +} + +// evalIdentifier evaluates IdentifierNode +func (j *JSONPath) evalIdentifier(input []reflect.Value, node *IdentifierNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + results := []reflect.Value{} + switch node.Name { + case "range": + j.beginRange++ + results = input + case "end": + if j.inRange > 0 { + j.endRange++ + } else { + return results, setFn, fmt.Errorf("not in range, nothing to end") + } + default: + return input, setFn, fmt.Errorf("unrecognized identifier %v", node.Name) + } + return results, setFn, nil +} + +// evalArray evaluates ArrayNode +func (j *JSONPath) evalArray(input []reflect.Value, node *ArrayNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + result := []reflect.Value{} + nextFns := []setFieldFunc{} + for k, value := range input { + + value, isNil := template.Indirect(value) + if isNil { + continue + } + if value.Kind() != reflect.Array && value.Kind() != reflect.Slice { + return input, nextFns, fmt.Errorf("%v is not array or slice", value.Type()) + } + params := node.Params + if !params[0].Known { + params[0].Value = 0 + } + if params[0].Value < 0 { + params[0].Value += value.Len() + } + if !params[1].Known { + params[1].Value = value.Len() + } + + if params[1].Value < 0 || (params[1].Value == 0 && params[1].Derived) { + params[1].Value += value.Len() + } + sliceLength := value.Len() + if params[1].Value != params[0].Value { // if you're requesting zero elements, allow it through. + if params[0].Value >= sliceLength || params[0].Value < 0 { + return input, nextFns, fmt.Errorf("array index out of bounds: index %d, length %d", params[0].Value, sliceLength) + } + if params[1].Value > sliceLength || params[1].Value < 0 { + return input, nextFns, fmt.Errorf("array index out of bounds: index %d, length %d", params[1].Value-1, sliceLength) + } + if params[0].Value > params[1].Value { + return input, nextFns, fmt.Errorf("starting index %d is greater than ending index %d", params[0].Value, params[1].Value) + } + } else { + return result, nextFns, nil + } + + value = value.Slice(params[0].Value, params[1].Value) + + step := 1 + if params[2].Known { + if params[2].Value <= 0 { + return input, nextFns, fmt.Errorf("step must be > 0") + } + step = params[2].Value + } + + loopResult := []reflect.Value{} + for i := 0; i < value.Len(); i += step { + loopResult = append(loopResult, value.Index(i)) + } + result = append(result, loopResult...) + + s := reflect.MakeSlice(value.Type(), len(loopResult), len(loopResult)) + for i := 0; i < len(loopResult); i++ { + ii := i + s.Index(ii).Set(loopResult[i]) + nextFns = append(nextFns, func(val reflect.Value) error { + s.Index(ii).Set(val) + return nil + }) + } + + setFn[k](s) + } + return result, nextFns, nil +} + +func (j *JSONPath) _evalUnion(input []reflect.Value, node *UnionNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + result := []reflect.Value{} + fns := []setFieldFunc{} + + union := make([][]reflect.Value, len(input)) + setFn_ := make([]setFieldFunc, len(input)) + + for i := 0; i < len(input); i++ { + ii := i + setFn_[i] = func(val reflect.Value) error { + union[ii] = append(union[ii], val) + return nil + } + } + + for _, listNode := range node.Nodes { + temp, nextFn, err := j._evalList(input, listNode, setFn_) + if err != nil { + return input, fns, err + } + result = append(result, temp...) + fns = append(fns, nextFn...) + } + + for i, fn := range setFn { + if len(union[i]) == 0 { + continue + } + + m := union[i][0] + for j := 1; j < len(union[i]); j++ { + val := union[i][j] + for _, key := range val.MapKeys() { + m.SetMapIndex(key, val.MapIndex(key)) + } + } + fn(m) + } + + return result, fns, nil +} + +// evalUnion evaluates UnionNode +func (j *JSONPath) evalUnion(input []reflect.Value, node *UnionNode) ([]reflect.Value, error) { + result := []reflect.Value{} + for _, listNode := range node.Nodes { + temp, err := j.evalList(input, listNode) + if err != nil { + return input, err + } + result = append(result, temp...) + } + return result, nil +} + +//lint:ignore U1000 ignore unused function +func (j *JSONPath) findFieldInValue(value *reflect.Value, node *FieldNode) (reflect.Value, error) { + t := value.Type() + var inlineValue *reflect.Value + for ix := 0; ix < t.NumField(); ix++ { + f := t.Field(ix) + jsonTag := f.Tag.Get("json") + parts := strings.Split(jsonTag, ",") + if len(parts) == 0 { + continue + } + if parts[0] == node.Value { + return value.Field(ix), nil + } + if len(parts[0]) == 0 { + val := value.Field(ix) + inlineValue = &val + } + } + if inlineValue != nil { + if inlineValue.Kind() == reflect.Struct { + // handle 'inline' + match, err := j.findFieldInValue(inlineValue, node) + if err != nil { + return reflect.Value{}, err + } + if match.IsValid() { + return match, nil + } + } + } + return value.FieldByName(node.Value), nil +} + +// evalField evaluates field of struct or key of map. +func (j *JSONPath) evalField(input []reflect.Value, node *FieldNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + results := []reflect.Value{} + nextFns := []setFieldFunc{} + // If there's no input, there's no output + if len(input) == 0 { + return results, nextFns, nil + } + for k, value := range input { + var result reflect.Value + var fn setFieldFunc + value, isNil := template.Indirect(value) + if isNil { + continue + } + + if value.Kind() != reflect.Map { + return results, nextFns, fmt.Errorf("%v is of the type %T, expected map[string]interface{}", value.Interface(), value.Interface()) + } else { + mapKeyType := value.Type().Key() + nodeValue := reflect.ValueOf(node.Value) + // node value type must be convertible to map key type + if !nodeValue.Type().ConvertibleTo(mapKeyType) { + return results, nextFns, fmt.Errorf("%s is not convertible to %s", nodeValue, mapKeyType) + } + key := nodeValue.Convert(mapKeyType) + result = value.MapIndex(key) + + val := reflect.MakeMap(value.Type()) + val.SetMapIndex(key, result) + setFn[k](val) + + fn = func(val_ reflect.Value) error { + val.SetMapIndex(key, val_) + return nil + } + } + + if result.IsValid() { + results = append(results, result) + nextFns = append(nextFns, fn) + } + } + if len(results) == 0 { + if j.allowMissingKeys { + return results, nextFns, nil + } + return results, nextFns, fmt.Errorf("%s is not found", node.Value) + } + return results, nextFns, nil +} + +// evalWildcard extracts all contents of the given value +func (j *JSONPath) evalWildcard(input []reflect.Value, _ *WildcardNode) ([]reflect.Value, error) { + results := []reflect.Value{} + for _, value := range input { + value, isNil := template.Indirect(value) + if isNil { + continue + } + + kind := value.Kind() + if kind == reflect.Struct { + for i := 0; i < value.NumField(); i++ { + results = append(results, value.Field(i)) + } + } else if kind == reflect.Map { + for _, key := range value.MapKeys() { + results = append(results, value.MapIndex(key)) + } + } else if kind == reflect.Array || kind == reflect.Slice || kind == reflect.String { + for i := 0; i < value.Len(); i++ { + results = append(results, value.Index(i)) + } + } + } + return results, nil +} + +// evalRecursive visits the given value recursively and pushes all of them to result +func (j *JSONPath) evalRecursive(input []reflect.Value, node *RecursiveNode) ([]reflect.Value, error) { + result := []reflect.Value{} + for _, value := range input { + results := []reflect.Value{} + value, isNil := template.Indirect(value) + if isNil { + continue + } + + kind := value.Kind() + if kind == reflect.Struct { + for i := 0; i < value.NumField(); i++ { + results = append(results, value.Field(i)) + } + } else if kind == reflect.Map { + for _, key := range value.MapKeys() { + results = append(results, value.MapIndex(key)) + } + } else if kind == reflect.Array || kind == reflect.Slice || kind == reflect.String { + for i := 0; i < value.Len(); i++ { + results = append(results, value.Index(i)) + } + } + if len(results) != 0 { + result = append(result, value) + output, err := j.evalRecursive(results, node) + if err != nil { + return result, err + } + result = append(result, output...) + } + } + return result, nil +} + +// evalFilter filters array according to FilterNode +func (j *JSONPath) evalFilter(input []reflect.Value, node *FilterNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { + results := []reflect.Value{} + fns := []setFieldFunc{} + for k, value := range input { + value, _ = template.Indirect(value) + + if value.Kind() != reflect.Array && value.Kind() != reflect.Slice { + return input, fns, fmt.Errorf("%v is not array or slice and cannot be filtered", value) + } + + loopResult := []reflect.Value{} + for i := 0; i < value.Len(); i++ { + temp := []reflect.Value{value.Index(i)} + lefts, err := j.evalList(temp, node.Left) + + // case exists + if node.Operator == "exists" { + if len(lefts) > 0 { + results = append(results, value.Index(i)) + } + continue + } + + if err != nil { + return input, fns, err + } + + var left, right interface{} + switch { + case len(lefts) == 0: + continue + case len(lefts) > 1: + return input, fns, fmt.Errorf("can only compare one element at a time") + } + left = lefts[0].Interface() + + rights, err := j.evalList(temp, node.Right) + if err != nil { + return input, fns, err + } + switch { + case len(rights) == 0: + continue + case len(rights) > 1: + return input, fns, fmt.Errorf("can only compare one element at a time") + } + right = rights[0].Interface() + + pass := false + switch node.Operator { + case "<": + pass, err = template.Less(left, right) + case ">": + pass, err = template.Greater(left, right) + case "==": + pass, err = template.Equal(left, right) + case "!=": + pass, err = template.NotEqual(left, right) + case "<=": + pass, err = template.LessEqual(left, right) + case ">=": + pass, err = template.GreaterEqual(left, right) + default: + return results, fns, fmt.Errorf("unrecognized filter operator %s", node.Operator) + } + if err != nil { + return results, fns, err + } + if pass { + loopResult = append(loopResult, value.Index(i)) + } + } + + s := reflect.MakeSlice(value.Type(), len(loopResult), len(loopResult)) + for i := 0; i < len(loopResult); i++ { + ii := i + s.Index(ii).Set(loopResult[i]) + fns = append(fns, func(val reflect.Value) error { + s.Index(ii).Set(val) + return nil + }) + } + + setFn[k](s) + results = append(results, loopResult...) + } + return results, fns, nil +} diff --git a/jsonextracter/jsonpath_test.go b/jsonextracter/jsonpath_test.go new file mode 100644 index 0000000..44efd97 --- /dev/null +++ b/jsonextracter/jsonpath_test.go @@ -0,0 +1,171 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "encoding/json" + "testing" +) + +type jsonPathTest struct { + name string + template string + input map[string]interface{} + expect string + expectError bool +} + +func (t *jsonPathTest) Prepare(allowMissingKeys bool) (*JSONPath, error) { + jp := New(t.name) + jp.AllowMissingKeys(allowMissingKeys) + return jp, jp.Parse(t.template) +} + +func benchmarkJSONPath(test jsonPathTest, allowMissingKeys bool, b *testing.B) { + jp, err := test.Prepare(allowMissingKeys) + if err != nil { + if !test.expectError { + b.Errorf("in %s, parse %s error %v", test.name, test.template, err) + return + } + } + + b.ResetTimer() + for n := 0; n < b.N; n++ { + jp.Extract(test.input) + } +} + +func testJSONPath(tests []jsonPathTest, allowMissingKeys bool, t *testing.T) { + for _, test := range tests { + jp, err := test.Prepare(allowMissingKeys) + if err != nil { + if !test.expectError { + t.Errorf("in %s, parse %s error %v", test.name, test.template, err) + continue + } + } + + got, err := jp.Extract(test.input) + + if test.expectError { + if err == nil { + t.Errorf(`in %s, expected execute error, got %q`, test.name, got) + } + } else if err != nil { + t.Errorf("in %s, execute error %v", test.name, err) + } + + bytes_, _ := json.Marshal(got) + out := string(bytes_) + + if out != test.expect { + t.Errorf(`in %s, expect to get "%s", got "%s"`, test.name, test.expect, out) + } + } +} + +var ( + pod = []byte(`{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "labels": { + "name": "pause", + "app": "pause" + }, + "name": "pause", + "namespace": "default" + }, + "spec": { + "containers": [ + { + "image": "registry.k8s.io/pause:3.8", + "imagePullPolicy": "IfNotPresent", + "name": "pause1", + "resources": { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + }, + { + "image": "registry.k8s.io/pause:3.8", + "imagePullPolicy": "IfNotPresent", + "name": "pause2", + "resources": { + "limits": { + "cpu": "10m", + "memory": "64Mi" + }, + "requests": { + "cpu": "10m", + "memory": "64Mi" + } + } + } + ] + } +}`) + + podData map[string]interface{} + + arbitrary = map[string]interface{}{ + "e": struct { + F1 string `json:"f1"` + F2 string `json:"f2"` + }{F1: "f1", F2: "f2"}, + } +) + +func init() { + json.Unmarshal(pod, &podData) +} + +func TestJSONPath(t *testing.T) { + podTests := []jsonPathTest{ + {"empty", ``, podData, `null`, false}, + {"containers name", `{.kind}`, podData, `{"kind":"Pod"}`, false}, + {"containers name", `{.spec.containers[*].name}`, podData, `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, false}, + {"containers name (range)", `{range .spec.containers[*]}{.name}{end}`, podData, `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, false}, + {"containers name and image", `{.spec.containers[*]['name', 'image']}`, podData, `{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","name":"pause1"},{"image":"registry.k8s.io/pause:3.8","name":"pause2"}]}}`, false}, + {"containers name and cpu", `{.spec.containers[*]['name', 'resources.requests.cpu']}`, podData, `{"spec":{"containers":[{"name":"pause1","resources":{"requests":{"cpu":"100m"}}},{"name":"pause2","resources":{"requests":{"cpu":"10m"}}}]}}`, false}, + {"container pause1 name and image", `{.spec.containers[?(@.name=="pause1")]['name', 'image']}`, podData, `{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","name":"pause1"}]}}`, false}, + {"pick one label", `{.metadata.labels.name}`, podData, `{"metadata":{"labels":{"name":"pause"}}}`, false}, + {"not exist label", `{.metadata.labels.xx.dd}`, podData, `null`, true}, + } + + testJSONPath(podTests, false, t) + + allowMissingTests := []jsonPathTest{ + {"containers image", `{.spec.containers[*]['xname', 'image']}`, podData, `{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8"},{"image":"registry.k8s.io/pause:3.8"}]}}`, false}, + {"not exist key", `{.spec.containers[*]['name', 'xx.dd']}`, podData, `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, false}, + {"not exist label", `{.metadata.labels.xx.dd}`, podData, `{"metadata":{"labels":{}}}`, false}, + } + + testJSONPath(allowMissingTests, true, t) +} + +func BenchmarkJSONPath(b *testing.B) { + t := jsonPathTest{"range nodes capacity", `{.kind}`, podData, `{"kind":"Pod"}`, false} + benchmarkJSONPath(t, true, b) +} diff --git a/jsonextracter/merge.go b/jsonextracter/merge.go new file mode 100644 index 0000000..1e66cf5 --- /dev/null +++ b/jsonextracter/merge.go @@ -0,0 +1,67 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "reflect" +) + +// Merge is a helper function that calls all extracters and merges their +// outputs by calling MergeFields. +func Merge(extracters []Extracter, input map[string]interface{}) (map[string]interface{}, error) { + var merged map[string]interface{} + + for _, ex := range extracters { + field, err := ex.Extract(input) + if err != nil { + return nil, err + } + + if merged == nil { + merged = field + } else { + merged = MergeFields(merged, field) + } + } + + return merged, nil +} + +// MergeFields merges src into dst. +// +// Note: the merge operation on two nested list is replacing. +func MergeFields(dst, src map[string]interface{}) map[string]interface{} { + for key, val := range src { + if cur, ok := dst[key]; ok { + if reflect.TypeOf(val) != reflect.TypeOf(cur) { + return nil + } + + switch cur := cur.(type) { + case []interface{}: + dst[key] = val.([]interface{}) + case map[string]interface{}: + dst[key] = MergeFields(cur, val.(map[string]interface{})) + default: + dst[key] = val + } + } else { + dst[key] = val + } + } + return dst +} diff --git a/jsonextracter/merge_test.go b/jsonextracter/merge_test.go new file mode 100644 index 0000000..ada0677 --- /dev/null +++ b/jsonextracter/merge_test.go @@ -0,0 +1,133 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jsonextracter + +import ( + "bytes" + "encoding/json" + "testing" + "text/template" + + "github.com/Masterminds/sprig/v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func BenchmarkJSONPathMerge(b *testing.B) { + tests := []jsonPathTest{ + {"kind", `{.kind}`, podData, "", false}, + {"apiVersion", "{.apiVersion}", podData, "", false}, + {"metadata", "{.metadata}", podData, "", false}, + } + + extracters := make([]Extracter, 0) + + for _, test := range tests { + ex, err := test.Prepare(false) + if err != nil { + if !test.expectError { + b.Errorf("in %s, parse %s error %v", test.name, test.template, err) + } + return + } + extracters = append(extracters, ex) + } + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + Merge(extracters, podData) + } +} + +func BenchmarkFieldPathMerge(b *testing.B) { + fields := []string{"kind", "apiVersion", "metadata"} + + extracters := make([]Extracter, 0) + + for _, f := range fields { + extracters = append(extracters, NewNestedFieldPath([]string{f}, false)) + } + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + Merge(extracters, podData) + } +} + +func BenchmarkTmpl(b *testing.B) { + tmpl := `{"kind": "{{ .Object.kind }}","apiVersion": "{{ .Object.apiVersion}}","metadata": {{ toJson .Object.metadata }}}` + obj := unstructured.Unstructured{Object: podData} + + t, _ := template.New("transformTemplate").Funcs(sprig.FuncMap()).Parse(tmpl) + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + var buf bytes.Buffer + t.Execute(&buf, obj) + + var dest unstructured.Unstructured + json.Unmarshal(buf.Bytes(), &dest) + } +} + +func TestMerge(t *testing.T) { + containerName := jsonPathTest{"containers name", `{.spec.containers[*].name}`, podData, "", false} + containerNameExtracter, _ := containerName.Prepare(true) + + containerImage := jsonPathTest{"containers image", `{.spec.containers[*].image}`, podData, "", false} + containerImageExtracter, _ := containerImage.Prepare(true) + + kindExtracter := NewNestedFieldPath([]string{"kind"}, true) + + apiVersionExtracter := NewNestedFieldPath([]string{"apiVersion"}, true) + + type args struct { + extracters []Extracter + input map[string]interface{} + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "merge name and image", args: args{extracters: []Extracter{containerImageExtracter, containerNameExtracter}, input: podData}, + want: `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, wantErr: false, + }, + { + name: "name kind apiVersion", args: args{extracters: []Extracter{containerNameExtracter, kindExtracter, apiVersionExtracter}, input: podData}, + want: `{"apiVersion":"v1","kind":"Pod","spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Merge(tt.args.extracters, tt.args.input) + if (err != nil) != tt.wantErr { + t.Errorf("Merge() error = %v, wantErr %v", err, tt.wantErr) + return + } + + data, _ := json.Marshal(got) + if string(data) != tt.want { + t.Errorf("Merge() = %v, want %v", string(data), tt.want) + } + }) + } +} From ea477a761b744d0f092a35443b61ee11de558a5c Mon Sep 17 00:00:00 2001 From: iamryanchia Date: Fri, 30 Aug 2024 15:32:02 +0800 Subject: [PATCH 2/4] style: rename package --- {jsonextracter => extracter}/README.md | 20 +++++++++---------- {jsonextracter => extracter}/alias.go | 2 +- {jsonextracter => extracter}/extracter.go | 2 +- .../extracter_test.go | 2 +- {jsonextracter => extracter}/fieldpath.go | 2 +- .../fieldpath_test.go | 2 +- {jsonextracter => extracter}/jsonpath.go | 2 +- {jsonextracter => extracter}/jsonpath_test.go | 2 +- {jsonextracter => extracter}/merge.go | 2 +- {jsonextracter => extracter}/merge_test.go | 2 +- 10 files changed, 19 insertions(+), 19 deletions(-) rename {jsonextracter => extracter}/README.md (68%) rename {jsonextracter => extracter}/alias.go (98%) rename {jsonextracter => extracter}/extracter.go (98%) rename {jsonextracter => extracter}/extracter_test.go (99%) rename {jsonextracter => extracter}/fieldpath.go (98%) rename {jsonextracter => extracter}/fieldpath_test.go (99%) rename {jsonextracter => extracter}/jsonpath.go (99%) rename {jsonextracter => extracter}/jsonpath_test.go (99%) rename {jsonextracter => extracter}/merge.go (98%) rename {jsonextracter => extracter}/merge_test.go (99%) diff --git a/jsonextracter/README.md b/extracter/README.md similarity index 68% rename from jsonextracter/README.md rename to extracter/README.md index 417dc65..f0dea98 100644 --- a/jsonextracter/README.md +++ b/extracter/README.md @@ -1,6 +1,6 @@ -# JSON Extracter +# Extracter -Extract specific field from JSON and **output not only the field value but also its upstream structure**. +Extract specific field from JSON-like data and **output not only the field value but also its upstream structure**. A typical use case is to trim k8s objects in `TransformingInformer` to save informer memory. @@ -17,7 +17,7 @@ import ( "encoding/json" "fmt" - "kusionstack.io/kube-utils/jsonextracter" + "kusionstack.io/kube-utils/extracter" ) var pod = []byte(`{ @@ -57,18 +57,18 @@ func main() { json.Unmarshal(pod, &podData) kindPath := "{.kind}" - kindExtracter, _ := jsonextracter.BuildExtracter(kindPath, false) + kindExtracter, _ := extracter.BuildExtracter(kindPath, false) kind, _ := kindExtracter.Extract(podData) printJSON(kind) nameImagePath := "{.spec.containers[*]['name', 'image']}" - nameImageExtracter, _ := jsonextracter.BuildExtracter(nameImagePath, false) + nameImageExtracter, _ := extracter.BuildExtracter(nameImagePath, false) nameImage, _ := nameImageExtracter.Extract(podData) printJSON(nameImage) - merged, _ := jsonextracter.Merge([]jsonextracter.Extracter{kindExtracter, nameImageExtracter}, podData) + merged, _ := extracter.Merge([]extracter.Extracter{kindExtracter, nameImageExtracter}, podData) printJSON(merged) } ``` @@ -83,19 +83,19 @@ Output: ## Note -The merge behavior of the `jsonextracter.Merge` on the list is replacing. Therefore, if you retrieve the container name and image separately and merge them, the resulting output will not contain the image. +The merge behavior of the `extracter.Merge` on the list is replacing. Therefore, if you retrieve the container name and image separately and merge them, the resulting output will not contain the image. Code: ```go ... namePath := "{.spec.containers[*].name}" - nameExtracter, _ := jsonextracter.BuildExtracter(namePath, false) + nameExtracter, _ := extracter.BuildExtracter(namePath, false) imagePath := "{.spec.containers[*].image}" - imageExtracter, _ := jsonextracter.BuildExtracter(imagePath, false) + imageExtracter, _ := extracter.BuildExtracter(imagePath, false) - merged, _ = jsonextracter.Merge([]jsonextracter.Extracter{imageExtracter, nameExtracter}, podData) + merged, _ = extracter.Merge([]extracter.Extracter{imageExtracter, nameExtracter}, podData) printJSON(merged) ... ``` diff --git a/jsonextracter/alias.go b/extracter/alias.go similarity index 98% rename from jsonextracter/alias.go rename to extracter/alias.go index 96e3df2..fe07efc 100644 --- a/jsonextracter/alias.go +++ b/extracter/alias.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "k8s.io/client-go/util/jsonpath" diff --git a/jsonextracter/extracter.go b/extracter/extracter.go similarity index 98% rename from jsonextracter/extracter.go rename to extracter/extracter.go index ba5b480..c13ad87 100644 --- a/jsonextracter/extracter.go +++ b/extracter/extracter.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "fmt" diff --git a/jsonextracter/extracter_test.go b/extracter/extracter_test.go similarity index 99% rename from jsonextracter/extracter_test.go rename to extracter/extracter_test.go index 70d9d21..ba63185 100644 --- a/jsonextracter/extracter_test.go +++ b/extracter/extracter_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "reflect" diff --git a/jsonextracter/fieldpath.go b/extracter/fieldpath.go similarity index 98% rename from jsonextracter/fieldpath.go rename to extracter/fieldpath.go index 3bea37b..c21a664 100644 --- a/jsonextracter/fieldpath.go +++ b/extracter/fieldpath.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "fmt" diff --git a/jsonextracter/fieldpath_test.go b/extracter/fieldpath_test.go similarity index 99% rename from jsonextracter/fieldpath_test.go rename to extracter/fieldpath_test.go index 9e03b7c..78a607b 100644 --- a/jsonextracter/fieldpath_test.go +++ b/extracter/fieldpath_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "encoding/json" diff --git a/jsonextracter/jsonpath.go b/extracter/jsonpath.go similarity index 99% rename from jsonextracter/jsonpath.go rename to extracter/jsonpath.go index 4c4620e..5b674f5 100644 --- a/jsonextracter/jsonpath.go +++ b/extracter/jsonpath.go @@ -17,7 +17,7 @@ // Copied and adapted from https://github.com/kubernetes/client-go/blob/master/util/jsonpath/jsonpath.go -package jsonextracter +package extracter import ( "fmt" diff --git a/jsonextracter/jsonpath_test.go b/extracter/jsonpath_test.go similarity index 99% rename from jsonextracter/jsonpath_test.go rename to extracter/jsonpath_test.go index 44efd97..b7945b7 100644 --- a/jsonextracter/jsonpath_test.go +++ b/extracter/jsonpath_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "encoding/json" diff --git a/jsonextracter/merge.go b/extracter/merge.go similarity index 98% rename from jsonextracter/merge.go rename to extracter/merge.go index 1e66cf5..a7f10d6 100644 --- a/jsonextracter/merge.go +++ b/extracter/merge.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "reflect" diff --git a/jsonextracter/merge_test.go b/extracter/merge_test.go similarity index 99% rename from jsonextracter/merge_test.go rename to extracter/merge_test.go index ada0677..e0938e9 100644 --- a/jsonextracter/merge_test.go +++ b/extracter/merge_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package jsonextracter +package extracter import ( "bytes" From 0d0efb7967b84d052aea774202bde0754b9b68b7 Mon Sep 17 00:00:00 2001 From: iamryanchia Date: Fri, 30 Aug 2024 17:49:21 +0800 Subject: [PATCH 3/4] refactor: reorganize export functions --- extracter/README.md | 15 +++--- extracter/alias.go | 2 - extracter/extracter.go | 92 +++++++++++++++++++++++++++++-------- extracter/extracter_test.go | 74 +++++++++++++++++++++++------ extracter/fieldpath.go | 14 +++--- extracter/jsonpath.go | 50 ++++++++++---------- extracter/jsonpath_test.go | 6 +-- extracter/merge.go | 21 --------- extracter/merge_test.go | 56 ++-------------------- 9 files changed, 181 insertions(+), 149 deletions(-) diff --git a/extracter/README.md b/extracter/README.md index f0dea98..29058ec 100644 --- a/extracter/README.md +++ b/extracter/README.md @@ -57,18 +57,19 @@ func main() { json.Unmarshal(pod, &podData) kindPath := "{.kind}" - kindExtracter, _ := extracter.BuildExtracter(kindPath, false) + kindExtracter, _ := extracter.New([]string{kindPath}, false) kind, _ := kindExtracter.Extract(podData) printJSON(kind) nameImagePath := "{.spec.containers[*]['name', 'image']}" - nameImageExtracter, _ := extracter.BuildExtracter(nameImagePath, false) + nameImageExtracter, _ := extracter.New([]string{nameImagePath}, false) nameImage, _ := nameImageExtracter.Extract(podData) printJSON(nameImage) - merged, _ := extracter.Merge([]extracter.Extracter{kindExtracter, nameImageExtracter}, podData) + mergeExtracter, _ := extracter.New([]string{kindPath, nameImagePath}, false) + merged, _ := mergeExtracter.Extract(podData) printJSON(merged) } ``` @@ -83,19 +84,17 @@ Output: ## Note -The merge behavior of the `extracter.Merge` on the list is replacing. Therefore, if you retrieve the container name and image separately and merge them, the resulting output will not contain the image. +The merge behavior on the list is replacing. Therefore, if you retrieve the container name and image separately and merge them, the resulting output will not contain the image. Code: ```go ... namePath := "{.spec.containers[*].name}" - nameExtracter, _ := extracter.BuildExtracter(namePath, false) - imagePath := "{.spec.containers[*].image}" - imageExtracter, _ := extracter.BuildExtracter(imagePath, false) - merged, _ = extracter.Merge([]extracter.Extracter{imageExtracter, nameExtracter}, podData) + mergeExtracter, _ = extracter.New([]string{imagePath, namePath}, false) + merged, _ = mergeExtracter.Extract(podData) printJSON(merged) ... ``` diff --git a/extracter/alias.go b/extracter/alias.go index fe07efc..f5fafc6 100644 --- a/extracter/alias.go +++ b/extracter/alias.go @@ -36,5 +36,3 @@ type ( UnionNode = jsonpath.UnionNode IdentifierNode = jsonpath.IdentifierNode ) - -var Parse = jsonpath.Parse diff --git a/extracter/extracter.go b/extracter/extracter.go index c13ad87..7439629 100644 --- a/extracter/extracter.go +++ b/extracter/extracter.go @@ -17,6 +17,7 @@ package extracter import ( + "errors" "fmt" "k8s.io/client-go/util/jsonpath" @@ -26,34 +27,87 @@ type Extracter interface { Extract(data map[string]interface{}) (map[string]interface{}, error) } -// BuildExtracter automatically determines whether to use FieldPathExtracter or JSONPathExtracter. -// If the input jsonPath only involves map operations, it will return FieldPathExtracter, -// as it has better performance. -func BuildExtracter(jsonPath string, allowMissingKeys bool) (Extracter, error) { - parser, err := Parse(jsonPath, jsonPath) +// parse is unlike the jsonpath.Parse, which supports multi-paths input. +// The input like `{.kind} {.apiVersion}` or +// `{range .spec.containers[*]}{.name}{end}` will result in an error. +func parse(name, text string) (*Parser, error) { + p, err := jsonpath.Parse(name, text) if err != nil { - return nil, fmt.Errorf("error in parsing path %q: %w", jsonPath, err) + return nil, err } - rootNodes := parser.Root.Nodes - if len(rootNodes) == 0 { - return NewNestedFieldPath(nil, allowMissingKeys), nil + if len(p.Root.Nodes) > 1 { + return nil, errors.New("not support multi-paths input") } - if len(rootNodes) == 1 { - nodes := rootNodes[0].(*jsonpath.ListNode).Nodes - fields := make([]string, 0, len(nodes)) - for _, node := range nodes { - if node.Type() == jsonpath.NodeField { - fields = append(fields, node.(*jsonpath.FieldNode).Value) + return p, nil +} + +// New creates an Extracter. For each jsonPaths, FieldPathExtracter will +// be parsed whenever possible, as it has better performance +func New(jsonPaths []string, allowMissingKeys bool) (Extracter, error) { + var extracters []Extracter + + for _, p := range jsonPaths { + parser, err := parse(p, p) + if err != nil { + return nil, fmt.Errorf("error in parsing path %q: %w", p, err) + } + + rootNodes := parser.Root.Nodes + if len(rootNodes) == 0 { + extracters = append(extracters, NewNestedFieldPathExtracter(nil, allowMissingKeys)) + continue + } + + if len(rootNodes) == 1 { + nodes := rootNodes[0].(*jsonpath.ListNode).Nodes + fields := make([]string, 0, len(nodes)) + for _, node := range nodes { + if node.Type() == jsonpath.NodeField { + fields = append(fields, node.(*jsonpath.FieldNode).Value) + } } + + if len(nodes) == len(fields) { + fp := NewNestedFieldPathExtracter(fields, allowMissingKeys) + extracters = append(extracters, fp) + continue + } + } + + jp := &JSONPathExtracter{name: parser.Name, parser: parser, allowMissingKeys: allowMissingKeys} + extracters = append(extracters, jp) + } + + if len(extracters) == 1 { + return extracters[0], nil + } + + return &Extracters{extracters}, nil +} + +// Extracters makes it easy when you want to extract multi fields and merge them. +type Extracters struct { + extracters []Extracter +} + +// Extract calls all extracters in order and merges their outputs by calling MergeFields. +func (e *Extracters) Extract(data map[string]interface{}) (map[string]interface{}, error) { + var merged map[string]interface{} + + for _, ex := range e.extracters { + field, err := ex.Extract(data) + if err != nil { + return nil, err } - if len(nodes) == len(fields) { - return NewNestedFieldPath(fields, allowMissingKeys), nil + if merged == nil { + merged = field + } else { + merged = MergeFields(merged, field) } } - jp := &JSONPath{name: parser.Name, parser: parser, allowMissingKeys: allowMissingKeys} - return jp, nil + return merged, nil } diff --git a/extracter/extracter_test.go b/extracter/extracter_test.go index ba63185..c12a525 100644 --- a/extracter/extracter_test.go +++ b/extracter/extracter_test.go @@ -17,13 +17,14 @@ package extracter import ( + "encoding/json" "reflect" "testing" ) -func TestBuildExtracter(t *testing.T) { +func TestNew(t *testing.T) { type args struct { - path string + paths []string allowMissingKeys bool } tests := []struct { @@ -32,25 +33,72 @@ func TestBuildExtracter(t *testing.T) { want Extracter wantErr bool }{ - {name: "invalid path", args: args{path: `{`, allowMissingKeys: false}, want: nil, wantErr: true}, - {name: "fieldPath extracter", args: args{path: `{}`, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, - {name: "fieldPath extracter", args: args{path: ``, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, - {name: "fieldPath extracter", args: args{path: `{.metadata.labels.name}`, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, - {name: "fieldPath extracter", args: args{path: `{.metadata.labels['name']}`, allowMissingKeys: false}, want: &NestedFieldPath{}, wantErr: false}, - {name: "jsonPath extracter", args: args{path: `{.metadata.labels.name}{.metadata.labels.app}`, allowMissingKeys: false}, want: &JSONPath{}, wantErr: false}, - {name: "jsonPath extracter", args: args{path: `{.metadata.labels['name', 'app']}`, allowMissingKeys: false}, want: &JSONPath{}, wantErr: false}, - {name: "jsonPath extracter", args: args{path: `{.spec.containers[*].name}`, allowMissingKeys: false}, want: &JSONPath{}, wantErr: false}, + {name: "invalid path", args: args{paths: []string{`{`}, allowMissingKeys: false}, want: nil, wantErr: true}, + {name: "fieldPath extracter", args: args{paths: []string{`{}`}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{``}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{`{.metadata.labels.name}`}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{`{.metadata.labels['name']}`}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, + {name: "jsonPath extracter", args: args{paths: []string{`{.metadata.labels.name}{.metadata.labels.app}`}, allowMissingKeys: false}, want: nil, wantErr: true}, + {name: "jsonPath extracter", args: args{paths: []string{`{.metadata.labels['name', 'app']}`}, allowMissingKeys: false}, want: &JSONPathExtracter{}, wantErr: false}, + {name: "jsonPath extracter", args: args{paths: []string{`{.spec.containers[*].name}`}, allowMissingKeys: false}, want: &JSONPathExtracter{}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := BuildExtracter(tt.args.path, tt.args.allowMissingKeys) + got, err := New(tt.args.paths, tt.args.allowMissingKeys) if (err != nil) != tt.wantErr { - t.Errorf("BuildExtracter() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) return } if reflect.TypeOf(tt.want) != reflect.TypeOf(got) { - t.Errorf("BuildExtracter() = %T, want %T", got, tt.want) + t.Errorf("New() = %T, want %T", got, tt.want) + } + }) + } +} + +func TestExtracters_Extract(t *testing.T) { + containerNamePath := `{.spec.containers[*].name}` + containerImagePath := `{.spec.containers[*].image}` + kindPath := "{.kind}" + apiVersionPath := "{.apiVersion}" + + type args struct { + paths []string + input map[string]interface{} + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "merge name and image", args: args{paths: []string{containerImagePath, containerNamePath}, input: podData}, + want: `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, wantErr: false, + }, + { + name: "name kind apiVersion", args: args{paths: []string{containerNamePath, kindPath, apiVersionPath}, input: podData}, + want: `{"apiVersion":"v1","kind":"Pod","spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ex, err := New(tt.args.paths, true) + if (err != nil) != tt.wantErr { + t.Errorf("Extracters_Extract() error = %v, wantErr %v", err, tt.wantErr) + return + } + + got, err := ex.Extract(tt.args.input) + if (err != nil) != tt.wantErr { + t.Errorf("Extracters_Extract() error = %v, wantErr %v", err, tt.wantErr) + return + } + + data, _ := json.Marshal(got) + if string(data) != tt.want { + t.Errorf("Extracters_Extract() = %v, want %v", string(data), tt.want) } }) } diff --git a/extracter/fieldpath.go b/extracter/fieldpath.go index c21a664..675e5c8 100644 --- a/extracter/fieldpath.go +++ b/extracter/fieldpath.go @@ -20,20 +20,20 @@ import ( "fmt" ) -// NewNestedFieldPath constructs a FieldPathExtracter. -func NewNestedFieldPath(nestedField []string, allowMissingKeys bool) *NestedFieldPath { - return &NestedFieldPath{nestedField: nestedField, allowMissingKeys: allowMissingKeys} +// NewNestedFieldPathExtracter constructs a FieldPathExtracter. +func NewNestedFieldPathExtracter(nestedField []string, allowMissingKeys bool) *NestedFieldPathExtracter { + return &NestedFieldPathExtracter{nestedField: nestedField, allowMissingKeys: allowMissingKeys} } -// NestedFieldPath is used to wrap NestedFieldNoCopy function as an Extracter. -type NestedFieldPath struct { +// NestedFieldPathExtracter is used to wrap NestedFieldNoCopy function as an Extracter. +type NestedFieldPathExtracter struct { nestedField []string allowMissingKeys bool } // Extract outputs the nestedField's value and its upstream structure. -func (f *NestedFieldPath) Extract(data map[string]interface{}) (map[string]interface{}, error) { - return NestedFieldNoCopy(data, f.allowMissingKeys, f.nestedField...) +func (n *NestedFieldPathExtracter) Extract(data map[string]interface{}) (map[string]interface{}, error) { + return NestedFieldNoCopy(data, n.allowMissingKeys, n.nestedField...) } // NestedFieldNoCopy is similar to JSONPath.Extract. The difference is that it diff --git a/extracter/jsonpath.go b/extracter/jsonpath.go index 5b674f5..329cf2c 100644 --- a/extracter/jsonpath.go +++ b/extracter/jsonpath.go @@ -27,7 +27,7 @@ import ( "k8s.io/client-go/third_party/forked/golang/template" ) -type JSONPath struct { +type JSONPathExtracter struct { name string parser *Parser beginRange int @@ -39,9 +39,9 @@ type JSONPath struct { allowMissingKeys bool } -// New creates a new JSONPath with the given name. -func New(name string) *JSONPath { - return &JSONPath{ +// NewJSONPathExtracter creates a new JSONPath with the given name. +func NewJSONPathExtracter(name string) *JSONPathExtracter { + return &JSONPathExtracter{ name: name, beginRange: 0, inRange: 0, @@ -51,15 +51,15 @@ func New(name string) *JSONPath { // AllowMissingKeys allows a caller to specify whether they want an error if a field or map key // cannot be located, or simply an empty result. The receiver is returned for chaining. -func (j *JSONPath) AllowMissingKeys(allow bool) *JSONPath { +func (j *JSONPathExtracter) AllowMissingKeys(allow bool) *JSONPathExtracter { j.allowMissingKeys = allow return j } // Parse parses the given template and returns an error. -func (j *JSONPath) Parse(text string) error { +func (j *JSONPathExtracter) Parse(text string) error { var err error - j.parser, err = Parse(j.name, text) + j.parser, err = parse(j.name, text) return err } @@ -80,7 +80,7 @@ func makeNopSetFieldFuncSlice(n int) []setFieldFunc { // // The data structure of the extracted field must be of type `map[string]interface{}`, // and `struct` is not supported (an error will be returned). -func (j *JSONPath) Extract(data map[string]interface{}) (map[string]interface{}, error) { +func (j *JSONPathExtracter) Extract(data map[string]interface{}) (map[string]interface{}, error) { container := struct{ Root reflect.Value }{} setFn := func(val reflect.Value) error { container.Root = val @@ -99,7 +99,7 @@ func (j *JSONPath) Extract(data map[string]interface{}) (map[string]interface{}, return container.Root.Interface().(map[string]interface{}), nil } -func (j *JSONPath) FindResults(data interface{}, setFn setFieldFunc) ([][]reflect.Value, error) { +func (j *JSONPathExtracter) FindResults(data interface{}, setFn setFieldFunc) ([][]reflect.Value, error) { if j.parser == nil { return nil, fmt.Errorf("%s is an incomplete jsonpath template", j.name) } @@ -159,7 +159,7 @@ func (j *JSONPath) FindResults(data interface{}, setFn setFieldFunc) ([][]reflec return fullResult, nil } -func (j *JSONPath) _walk(value []reflect.Value, node Node, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) _walk(value []reflect.Value, node Node, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { switch node := node.(type) { case *ListNode: return j._evalList(value, node, setFn) @@ -179,7 +179,7 @@ func (j *JSONPath) _walk(value []reflect.Value, node Node, setFn []setFieldFunc) } // walk visits tree rooted at the given node in DFS order -func (j *JSONPath) walk(value []reflect.Value, node Node) ([]reflect.Value, error) { +func (j *JSONPathExtracter) walk(value []reflect.Value, node Node) ([]reflect.Value, error) { switch node := node.(type) { case *ListNode: return j.evalList(value, node) @@ -215,7 +215,7 @@ func (j *JSONPath) walk(value []reflect.Value, node Node) ([]reflect.Value, erro } // evalInt evaluates IntNode -func (j *JSONPath) evalInt(input []reflect.Value, node *IntNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalInt(input []reflect.Value, node *IntNode) ([]reflect.Value, error) { result := make([]reflect.Value, len(input)) for i := range input { result[i] = reflect.ValueOf(node.Value) @@ -224,7 +224,7 @@ func (j *JSONPath) evalInt(input []reflect.Value, node *IntNode) ([]reflect.Valu } // evalFloat evaluates FloatNode -func (j *JSONPath) evalFloat(input []reflect.Value, node *FloatNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalFloat(input []reflect.Value, node *FloatNode) ([]reflect.Value, error) { result := make([]reflect.Value, len(input)) for i := range input { result[i] = reflect.ValueOf(node.Value) @@ -233,7 +233,7 @@ func (j *JSONPath) evalFloat(input []reflect.Value, node *FloatNode) ([]reflect. } // evalBool evaluates BoolNode -func (j *JSONPath) evalBool(input []reflect.Value, node *BoolNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalBool(input []reflect.Value, node *BoolNode) ([]reflect.Value, error) { result := make([]reflect.Value, len(input)) for i := range input { result[i] = reflect.ValueOf(node.Value) @@ -241,7 +241,7 @@ func (j *JSONPath) evalBool(input []reflect.Value, node *BoolNode) ([]reflect.Va return result, nil } -func (j *JSONPath) _evalList(value []reflect.Value, node *ListNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) _evalList(value []reflect.Value, node *ListNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { var err error curValue := value curFns := setFn @@ -256,7 +256,7 @@ func (j *JSONPath) _evalList(value []reflect.Value, node *ListNode, setFn []setF } // evalList evaluates ListNode -func (j *JSONPath) evalList(value []reflect.Value, node *ListNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalList(value []reflect.Value, node *ListNode) ([]reflect.Value, error) { var err error curValue := value for _, node := range node.Nodes { @@ -269,7 +269,7 @@ func (j *JSONPath) evalList(value []reflect.Value, node *ListNode) ([]reflect.Va } // evalIdentifier evaluates IdentifierNode -func (j *JSONPath) evalIdentifier(input []reflect.Value, node *IdentifierNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) evalIdentifier(input []reflect.Value, node *IdentifierNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { results := []reflect.Value{} switch node.Name { case "range": @@ -288,7 +288,7 @@ func (j *JSONPath) evalIdentifier(input []reflect.Value, node *IdentifierNode, s } // evalArray evaluates ArrayNode -func (j *JSONPath) evalArray(input []reflect.Value, node *ArrayNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) evalArray(input []reflect.Value, node *ArrayNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { result := []reflect.Value{} nextFns := []setFieldFunc{} for k, value := range input { @@ -360,7 +360,7 @@ func (j *JSONPath) evalArray(input []reflect.Value, node *ArrayNode, setFn []set return result, nextFns, nil } -func (j *JSONPath) _evalUnion(input []reflect.Value, node *UnionNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) _evalUnion(input []reflect.Value, node *UnionNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { result := []reflect.Value{} fns := []setFieldFunc{} @@ -403,7 +403,7 @@ func (j *JSONPath) _evalUnion(input []reflect.Value, node *UnionNode, setFn []se } // evalUnion evaluates UnionNode -func (j *JSONPath) evalUnion(input []reflect.Value, node *UnionNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalUnion(input []reflect.Value, node *UnionNode) ([]reflect.Value, error) { result := []reflect.Value{} for _, listNode := range node.Nodes { temp, err := j.evalList(input, listNode) @@ -416,7 +416,7 @@ func (j *JSONPath) evalUnion(input []reflect.Value, node *UnionNode) ([]reflect. } //lint:ignore U1000 ignore unused function -func (j *JSONPath) findFieldInValue(value *reflect.Value, node *FieldNode) (reflect.Value, error) { +func (j *JSONPathExtracter) findFieldInValue(value *reflect.Value, node *FieldNode) (reflect.Value, error) { t := value.Type() var inlineValue *reflect.Value for ix := 0; ix < t.NumField(); ix++ { @@ -450,7 +450,7 @@ func (j *JSONPath) findFieldInValue(value *reflect.Value, node *FieldNode) (refl } // evalField evaluates field of struct or key of map. -func (j *JSONPath) evalField(input []reflect.Value, node *FieldNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) evalField(input []reflect.Value, node *FieldNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { results := []reflect.Value{} nextFns := []setFieldFunc{} // If there's no input, there's no output @@ -502,7 +502,7 @@ func (j *JSONPath) evalField(input []reflect.Value, node *FieldNode, setFn []set } // evalWildcard extracts all contents of the given value -func (j *JSONPath) evalWildcard(input []reflect.Value, _ *WildcardNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalWildcard(input []reflect.Value, _ *WildcardNode) ([]reflect.Value, error) { results := []reflect.Value{} for _, value := range input { value, isNil := template.Indirect(value) @@ -529,7 +529,7 @@ func (j *JSONPath) evalWildcard(input []reflect.Value, _ *WildcardNode) ([]refle } // evalRecursive visits the given value recursively and pushes all of them to result -func (j *JSONPath) evalRecursive(input []reflect.Value, node *RecursiveNode) ([]reflect.Value, error) { +func (j *JSONPathExtracter) evalRecursive(input []reflect.Value, node *RecursiveNode) ([]reflect.Value, error) { result := []reflect.Value{} for _, value := range input { results := []reflect.Value{} @@ -565,7 +565,7 @@ func (j *JSONPath) evalRecursive(input []reflect.Value, node *RecursiveNode) ([] } // evalFilter filters array according to FilterNode -func (j *JSONPath) evalFilter(input []reflect.Value, node *FilterNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *JSONPathExtracter) evalFilter(input []reflect.Value, node *FilterNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { results := []reflect.Value{} fns := []setFieldFunc{} for k, value := range input { diff --git a/extracter/jsonpath_test.go b/extracter/jsonpath_test.go index b7945b7..0aca18b 100644 --- a/extracter/jsonpath_test.go +++ b/extracter/jsonpath_test.go @@ -29,8 +29,8 @@ type jsonPathTest struct { expectError bool } -func (t *jsonPathTest) Prepare(allowMissingKeys bool) (*JSONPath, error) { - jp := New(t.name) +func (t *jsonPathTest) Prepare(allowMissingKeys bool) (*JSONPathExtracter, error) { + jp := NewJSONPathExtracter(t.name) jp.AllowMissingKeys(allowMissingKeys) return jp, jp.Parse(t.template) } @@ -146,7 +146,7 @@ func TestJSONPath(t *testing.T) { {"empty", ``, podData, `null`, false}, {"containers name", `{.kind}`, podData, `{"kind":"Pod"}`, false}, {"containers name", `{.spec.containers[*].name}`, podData, `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, false}, - {"containers name (range)", `{range .spec.containers[*]}{.name}{end}`, podData, `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, false}, + {"containers name (range)", `{range .spec.containers[*]}{.name}{end}`, podData, `null`, true}, {"containers name and image", `{.spec.containers[*]['name', 'image']}`, podData, `{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","name":"pause1"},{"image":"registry.k8s.io/pause:3.8","name":"pause2"}]}}`, false}, {"containers name and cpu", `{.spec.containers[*]['name', 'resources.requests.cpu']}`, podData, `{"spec":{"containers":[{"name":"pause1","resources":{"requests":{"cpu":"100m"}}},{"name":"pause2","resources":{"requests":{"cpu":"10m"}}}]}}`, false}, {"container pause1 name and image", `{.spec.containers[?(@.name=="pause1")]['name', 'image']}`, podData, `{"spec":{"containers":[{"image":"registry.k8s.io/pause:3.8","name":"pause1"}]}}`, false}, diff --git a/extracter/merge.go b/extracter/merge.go index a7f10d6..f9ebf0e 100644 --- a/extracter/merge.go +++ b/extracter/merge.go @@ -20,27 +20,6 @@ import ( "reflect" ) -// Merge is a helper function that calls all extracters and merges their -// outputs by calling MergeFields. -func Merge(extracters []Extracter, input map[string]interface{}) (map[string]interface{}, error) { - var merged map[string]interface{} - - for _, ex := range extracters { - field, err := ex.Extract(input) - if err != nil { - return nil, err - } - - if merged == nil { - merged = field - } else { - merged = MergeFields(merged, field) - } - } - - return merged, nil -} - // MergeFields merges src into dst. // // Note: the merge operation on two nested list is replacing. diff --git a/extracter/merge_test.go b/extracter/merge_test.go index e0938e9..168d67b 100644 --- a/extracter/merge_test.go +++ b/extracter/merge_test.go @@ -34,7 +34,6 @@ func BenchmarkJSONPathMerge(b *testing.B) { } extracters := make([]Extracter, 0) - for _, test := range tests { ex, err := test.Prepare(false) if err != nil { @@ -46,10 +45,11 @@ func BenchmarkJSONPathMerge(b *testing.B) { extracters = append(extracters, ex) } + ex := Extracters{extracters: extracters} b.ResetTimer() for n := 0; n < b.N; n++ { - Merge(extracters, podData) + ex.Extract(podData) } } @@ -57,15 +57,15 @@ func BenchmarkFieldPathMerge(b *testing.B) { fields := []string{"kind", "apiVersion", "metadata"} extracters := make([]Extracter, 0) - for _, f := range fields { - extracters = append(extracters, NewNestedFieldPath([]string{f}, false)) + extracters = append(extracters, NewNestedFieldPathExtracter([]string{f}, false)) } + ex := Extracters{extracters: extracters} b.ResetTimer() for n := 0; n < b.N; n++ { - Merge(extracters, podData) + ex.Extract(podData) } } @@ -85,49 +85,3 @@ func BenchmarkTmpl(b *testing.B) { json.Unmarshal(buf.Bytes(), &dest) } } - -func TestMerge(t *testing.T) { - containerName := jsonPathTest{"containers name", `{.spec.containers[*].name}`, podData, "", false} - containerNameExtracter, _ := containerName.Prepare(true) - - containerImage := jsonPathTest{"containers image", `{.spec.containers[*].image}`, podData, "", false} - containerImageExtracter, _ := containerImage.Prepare(true) - - kindExtracter := NewNestedFieldPath([]string{"kind"}, true) - - apiVersionExtracter := NewNestedFieldPath([]string{"apiVersion"}, true) - - type args struct { - extracters []Extracter - input map[string]interface{} - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - { - name: "merge name and image", args: args{extracters: []Extracter{containerImageExtracter, containerNameExtracter}, input: podData}, - want: `{"spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, wantErr: false, - }, - { - name: "name kind apiVersion", args: args{extracters: []Extracter{containerNameExtracter, kindExtracter, apiVersionExtracter}, input: podData}, - want: `{"apiVersion":"v1","kind":"Pod","spec":{"containers":[{"name":"pause1"},{"name":"pause2"}]}}`, wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := Merge(tt.args.extracters, tt.args.input) - if (err != nil) != tt.wantErr { - t.Errorf("Merge() error = %v, wantErr %v", err, tt.wantErr) - return - } - - data, _ := json.Marshal(got) - if string(data) != tt.want { - t.Errorf("Merge() = %v, want %v", string(data), tt.want) - } - }) - } -} From 26b3969145a2ca733333160d625403dc64485ac2 Mon Sep 17 00:00:00 2001 From: iamryanchia Date: Wed, 4 Sep 2024 15:47:57 +0800 Subject: [PATCH 4/4] refactor: hide unnecessary export functions --- extracter/extracter.go | 25 ++----- extracter/extracter_test.go | 12 ++-- extracter/fieldpath.go | 10 +-- extracter/jsonpath.go | 110 ++++++++++++++---------------- extracter/jsonpath_test.go | 13 ++-- extracter/merge.go | 6 +- extracter/{alias.go => parser.go} | 37 +++++----- extracter/parser_test.go | 50 ++++++++++++++ 8 files changed, 148 insertions(+), 115 deletions(-) rename extracter/{alias.go => parser.go} (56%) create mode 100644 extracter/parser_test.go diff --git a/extracter/extracter.go b/extracter/extracter.go index 7439629..f20b645 100644 --- a/extracter/extracter.go +++ b/extracter/extracter.go @@ -17,7 +17,6 @@ package extracter import ( - "errors" "fmt" "k8s.io/client-go/util/jsonpath" @@ -27,29 +26,13 @@ type Extracter interface { Extract(data map[string]interface{}) (map[string]interface{}, error) } -// parse is unlike the jsonpath.Parse, which supports multi-paths input. -// The input like `{.kind} {.apiVersion}` or -// `{range .spec.containers[*]}{.name}{end}` will result in an error. -func parse(name, text string) (*Parser, error) { - p, err := jsonpath.Parse(name, text) - if err != nil { - return nil, err - } - - if len(p.Root.Nodes) > 1 { - return nil, errors.New("not support multi-paths input") - } - - return p, nil -} - // New creates an Extracter. For each jsonPaths, FieldPathExtracter will // be parsed whenever possible, as it has better performance func New(jsonPaths []string, allowMissingKeys bool) (Extracter, error) { var extracters []Extracter for _, p := range jsonPaths { - parser, err := parse(p, p) + parser, err := Parse(p, p) if err != nil { return nil, fmt.Errorf("error in parsing path %q: %w", p, err) } @@ -76,7 +59,7 @@ func New(jsonPaths []string, allowMissingKeys bool) (Extracter, error) { } } - jp := &JSONPathExtracter{name: parser.Name, parser: parser, allowMissingKeys: allowMissingKeys} + jp := &jsonPathExtracter{name: parser.Name, parser: parser, allowMissingKeys: allowMissingKeys} extracters = append(extracters, jp) } @@ -92,7 +75,7 @@ type Extracters struct { extracters []Extracter } -// Extract calls all extracters in order and merges their outputs by calling MergeFields. +// Extract calls all extracters in order and merges their outputs by calling mergeFields. func (e *Extracters) Extract(data map[string]interface{}) (map[string]interface{}, error) { var merged map[string]interface{} @@ -105,7 +88,7 @@ func (e *Extracters) Extract(data map[string]interface{}) (map[string]interface{ if merged == nil { merged = field } else { - merged = MergeFields(merged, field) + merged = mergeFields(merged, field) } } diff --git a/extracter/extracter_test.go b/extracter/extracter_test.go index c12a525..fe20732 100644 --- a/extracter/extracter_test.go +++ b/extracter/extracter_test.go @@ -34,13 +34,13 @@ func TestNew(t *testing.T) { wantErr bool }{ {name: "invalid path", args: args{paths: []string{`{`}, allowMissingKeys: false}, want: nil, wantErr: true}, - {name: "fieldPath extracter", args: args{paths: []string{`{}`}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, - {name: "fieldPath extracter", args: args{paths: []string{``}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, - {name: "fieldPath extracter", args: args{paths: []string{`{.metadata.labels.name}`}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, - {name: "fieldPath extracter", args: args{paths: []string{`{.metadata.labels['name']}`}, allowMissingKeys: false}, want: &NestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{`{}`}, allowMissingKeys: false}, want: &nestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{``}, allowMissingKeys: false}, want: &nestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{`{.metadata.labels.name}`}, allowMissingKeys: false}, want: &nestedFieldPathExtracter{}, wantErr: false}, + {name: "fieldPath extracter", args: args{paths: []string{`{.metadata.labels['name']}`}, allowMissingKeys: false}, want: &nestedFieldPathExtracter{}, wantErr: false}, {name: "jsonPath extracter", args: args{paths: []string{`{.metadata.labels.name}{.metadata.labels.app}`}, allowMissingKeys: false}, want: nil, wantErr: true}, - {name: "jsonPath extracter", args: args{paths: []string{`{.metadata.labels['name', 'app']}`}, allowMissingKeys: false}, want: &JSONPathExtracter{}, wantErr: false}, - {name: "jsonPath extracter", args: args{paths: []string{`{.spec.containers[*].name}`}, allowMissingKeys: false}, want: &JSONPathExtracter{}, wantErr: false}, + {name: "jsonPath extracter", args: args{paths: []string{`{.metadata.labels['name', 'app']}`}, allowMissingKeys: false}, want: &jsonPathExtracter{}, wantErr: false}, + {name: "jsonPath extracter", args: args{paths: []string{`{.spec.containers[*].name}`}, allowMissingKeys: false}, want: &jsonPathExtracter{}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/extracter/fieldpath.go b/extracter/fieldpath.go index 675e5c8..7aaa2ee 100644 --- a/extracter/fieldpath.go +++ b/extracter/fieldpath.go @@ -21,18 +21,18 @@ import ( ) // NewNestedFieldPathExtracter constructs a FieldPathExtracter. -func NewNestedFieldPathExtracter(nestedField []string, allowMissingKeys bool) *NestedFieldPathExtracter { - return &NestedFieldPathExtracter{nestedField: nestedField, allowMissingKeys: allowMissingKeys} +func NewNestedFieldPathExtracter(nestedField []string, allowMissingKeys bool) Extracter { + return &nestedFieldPathExtracter{nestedField: nestedField, allowMissingKeys: allowMissingKeys} } -// NestedFieldPathExtracter is used to wrap NestedFieldNoCopy function as an Extracter. -type NestedFieldPathExtracter struct { +// nestedFieldPathExtracter is used to wrap NestedFieldNoCopy function as an Extracter. +type nestedFieldPathExtracter struct { nestedField []string allowMissingKeys bool } // Extract outputs the nestedField's value and its upstream structure. -func (n *NestedFieldPathExtracter) Extract(data map[string]interface{}) (map[string]interface{}, error) { +func (n *nestedFieldPathExtracter) Extract(data map[string]interface{}) (map[string]interface{}, error) { return NestedFieldNoCopy(data, n.allowMissingKeys, n.nestedField...) } diff --git a/extracter/jsonpath.go b/extracter/jsonpath.go index 329cf2c..6a33bf1 100644 --- a/extracter/jsonpath.go +++ b/extracter/jsonpath.go @@ -25,42 +25,32 @@ import ( "strings" "k8s.io/client-go/third_party/forked/golang/template" + "k8s.io/client-go/util/jsonpath" ) -type JSONPathExtracter struct { +type jsonPathExtracter struct { name string - parser *Parser + parser *parser beginRange int inRange int endRange int - lastEndNode *Node + lastEndNode *jsonpath.Node allowMissingKeys bool } -// NewJSONPathExtracter creates a new JSONPath with the given name. -func NewJSONPathExtracter(name string) *JSONPathExtracter { - return &JSONPathExtracter{ - name: name, +// NewJSONPathExtracter creates a new JSONPathExtracter with the given parser. +func NewJSONPathExtracter(parser *parser, allowMissingKeys bool) Extracter { + return &jsonPathExtracter{ + name: parser.Name, beginRange: 0, inRange: 0, endRange: 0, - } -} - -// AllowMissingKeys allows a caller to specify whether they want an error if a field or map key -// cannot be located, or simply an empty result. The receiver is returned for chaining. -func (j *JSONPathExtracter) AllowMissingKeys(allow bool) *JSONPathExtracter { - j.allowMissingKeys = allow - return j -} -// Parse parses the given template and returns an error. -func (j *JSONPathExtracter) Parse(text string) error { - var err error - j.parser, err = parse(j.name, text) - return err + parser: parser, + allowMissingKeys: allowMissingKeys, + } } type setFieldFunc func(val reflect.Value) error @@ -80,14 +70,14 @@ func makeNopSetFieldFuncSlice(n int) []setFieldFunc { // // The data structure of the extracted field must be of type `map[string]interface{}`, // and `struct` is not supported (an error will be returned). -func (j *JSONPathExtracter) Extract(data map[string]interface{}) (map[string]interface{}, error) { +func (j *jsonPathExtracter) Extract(data map[string]interface{}) (map[string]interface{}, error) { container := struct{ Root reflect.Value }{} setFn := func(val reflect.Value) error { container.Root = val return nil } - _, err := j.FindResults(data, setFn) + _, err := j.findResults(data, setFn) if err != nil { return nil, err } @@ -99,7 +89,7 @@ func (j *JSONPathExtracter) Extract(data map[string]interface{}) (map[string]int return container.Root.Interface().(map[string]interface{}), nil } -func (j *JSONPathExtracter) FindResults(data interface{}, setFn setFieldFunc) ([][]reflect.Value, error) { +func (j *jsonPathExtracter) findResults(data interface{}, setFn setFieldFunc) ([][]reflect.Value, error) { if j.parser == nil { return nil, fmt.Errorf("%s is an incomplete jsonpath template", j.name) } @@ -128,7 +118,7 @@ func (j *JSONPathExtracter) FindResults(data interface{}, setFn setFieldFunc) ([ if len(results) > 0 { for ri, value := range results { j.parser.Root.Nodes = nodes[i+1:] - nextResults, err := j.FindResults(value.Interface(), fn[ri]) + nextResults, err := j.findResults(value.Interface(), fn[ri]) if err != nil { return nil, err } @@ -138,7 +128,7 @@ func (j *JSONPathExtracter) FindResults(data interface{}, setFn setFieldFunc) ([ // If the range has no results, we still need to process the nodes within the range // so the position will advance to the end node j.parser.Root.Nodes = nodes[i+1:] - _, err := j.FindResults(nil, nopSetFieldFunc) + _, err := j.findResults(nil, nopSetFieldFunc) if err != nil { return nil, err } @@ -159,19 +149,19 @@ func (j *JSONPathExtracter) FindResults(data interface{}, setFn setFieldFunc) ([ return fullResult, nil } -func (j *JSONPathExtracter) _walk(value []reflect.Value, node Node, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) _walk(value []reflect.Value, node jsonpath.Node, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { switch node := node.(type) { - case *ListNode: + case *jsonpath.ListNode: return j._evalList(value, node, setFn) - case *FieldNode: + case *jsonpath.FieldNode: return j.evalField(value, node, setFn) - case *ArrayNode: + case *jsonpath.ArrayNode: return j.evalArray(value, node, setFn) - case *IdentifierNode: + case *jsonpath.IdentifierNode: return j.evalIdentifier(value, node, setFn) - case *UnionNode: + case *jsonpath.UnionNode: return j._evalUnion(value, node, setFn) - case *FilterNode: + case *jsonpath.FilterNode: return j.evalFilter(value, node, setFn) default: return nil, nil, fmt.Errorf("Extract does not support node %v", node) @@ -179,34 +169,34 @@ func (j *JSONPathExtracter) _walk(value []reflect.Value, node Node, setFn []setF } // walk visits tree rooted at the given node in DFS order -func (j *JSONPathExtracter) walk(value []reflect.Value, node Node) ([]reflect.Value, error) { +func (j *jsonPathExtracter) walk(value []reflect.Value, node jsonpath.Node) ([]reflect.Value, error) { switch node := node.(type) { - case *ListNode: + case *jsonpath.ListNode: return j.evalList(value, node) - case *TextNode: + case *jsonpath.TextNode: return []reflect.Value{reflect.ValueOf(node.Text)}, nil - case *FieldNode: + case *jsonpath.FieldNode: value, _, err := j.evalField(value, node, makeNopSetFieldFuncSlice(len(value))) return value, err - case *ArrayNode: + case *jsonpath.ArrayNode: value, _, err := j.evalArray(value, node, makeNopSetFieldFuncSlice(len(value))) return value, err - case *FilterNode: + case *jsonpath.FilterNode: value, _, err := j.evalFilter(value, node, makeNopSetFieldFuncSlice(len(value))) return value, err - case *IntNode: + case *jsonpath.IntNode: return j.evalInt(value, node) - case *BoolNode: + case *jsonpath.BoolNode: return j.evalBool(value, node) - case *FloatNode: + case *jsonpath.FloatNode: return j.evalFloat(value, node) - case *WildcardNode: + case *jsonpath.WildcardNode: return j.evalWildcard(value, node) - case *RecursiveNode: + case *jsonpath.RecursiveNode: return j.evalRecursive(value, node) - case *UnionNode: + case *jsonpath.UnionNode: return j.evalUnion(value, node) - case *IdentifierNode: + case *jsonpath.IdentifierNode: value, _, err := j.evalIdentifier(value, node, makeNopSetFieldFuncSlice(len(value))) return value, err default: @@ -215,7 +205,7 @@ func (j *JSONPathExtracter) walk(value []reflect.Value, node Node) ([]reflect.Va } // evalInt evaluates IntNode -func (j *JSONPathExtracter) evalInt(input []reflect.Value, node *IntNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalInt(input []reflect.Value, node *jsonpath.IntNode) ([]reflect.Value, error) { result := make([]reflect.Value, len(input)) for i := range input { result[i] = reflect.ValueOf(node.Value) @@ -224,7 +214,7 @@ func (j *JSONPathExtracter) evalInt(input []reflect.Value, node *IntNode) ([]ref } // evalFloat evaluates FloatNode -func (j *JSONPathExtracter) evalFloat(input []reflect.Value, node *FloatNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalFloat(input []reflect.Value, node *jsonpath.FloatNode) ([]reflect.Value, error) { result := make([]reflect.Value, len(input)) for i := range input { result[i] = reflect.ValueOf(node.Value) @@ -233,7 +223,7 @@ func (j *JSONPathExtracter) evalFloat(input []reflect.Value, node *FloatNode) ([ } // evalBool evaluates BoolNode -func (j *JSONPathExtracter) evalBool(input []reflect.Value, node *BoolNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalBool(input []reflect.Value, node *jsonpath.BoolNode) ([]reflect.Value, error) { result := make([]reflect.Value, len(input)) for i := range input { result[i] = reflect.ValueOf(node.Value) @@ -241,7 +231,7 @@ func (j *JSONPathExtracter) evalBool(input []reflect.Value, node *BoolNode) ([]r return result, nil } -func (j *JSONPathExtracter) _evalList(value []reflect.Value, node *ListNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) _evalList(value []reflect.Value, node *jsonpath.ListNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { var err error curValue := value curFns := setFn @@ -256,7 +246,7 @@ func (j *JSONPathExtracter) _evalList(value []reflect.Value, node *ListNode, set } // evalList evaluates ListNode -func (j *JSONPathExtracter) evalList(value []reflect.Value, node *ListNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalList(value []reflect.Value, node *jsonpath.ListNode) ([]reflect.Value, error) { var err error curValue := value for _, node := range node.Nodes { @@ -269,7 +259,7 @@ func (j *JSONPathExtracter) evalList(value []reflect.Value, node *ListNode) ([]r } // evalIdentifier evaluates IdentifierNode -func (j *JSONPathExtracter) evalIdentifier(input []reflect.Value, node *IdentifierNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) evalIdentifier(input []reflect.Value, node *jsonpath.IdentifierNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { results := []reflect.Value{} switch node.Name { case "range": @@ -288,7 +278,7 @@ func (j *JSONPathExtracter) evalIdentifier(input []reflect.Value, node *Identifi } // evalArray evaluates ArrayNode -func (j *JSONPathExtracter) evalArray(input []reflect.Value, node *ArrayNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) evalArray(input []reflect.Value, node *jsonpath.ArrayNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { result := []reflect.Value{} nextFns := []setFieldFunc{} for k, value := range input { @@ -360,7 +350,7 @@ func (j *JSONPathExtracter) evalArray(input []reflect.Value, node *ArrayNode, se return result, nextFns, nil } -func (j *JSONPathExtracter) _evalUnion(input []reflect.Value, node *UnionNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) _evalUnion(input []reflect.Value, node *jsonpath.UnionNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { result := []reflect.Value{} fns := []setFieldFunc{} @@ -403,7 +393,7 @@ func (j *JSONPathExtracter) _evalUnion(input []reflect.Value, node *UnionNode, s } // evalUnion evaluates UnionNode -func (j *JSONPathExtracter) evalUnion(input []reflect.Value, node *UnionNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalUnion(input []reflect.Value, node *jsonpath.UnionNode) ([]reflect.Value, error) { result := []reflect.Value{} for _, listNode := range node.Nodes { temp, err := j.evalList(input, listNode) @@ -416,7 +406,7 @@ func (j *JSONPathExtracter) evalUnion(input []reflect.Value, node *UnionNode) ([ } //lint:ignore U1000 ignore unused function -func (j *JSONPathExtracter) findFieldInValue(value *reflect.Value, node *FieldNode) (reflect.Value, error) { +func (j *jsonPathExtracter) findFieldInValue(value *reflect.Value, node *jsonpath.FieldNode) (reflect.Value, error) { t := value.Type() var inlineValue *reflect.Value for ix := 0; ix < t.NumField(); ix++ { @@ -450,7 +440,7 @@ func (j *JSONPathExtracter) findFieldInValue(value *reflect.Value, node *FieldNo } // evalField evaluates field of struct or key of map. -func (j *JSONPathExtracter) evalField(input []reflect.Value, node *FieldNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) evalField(input []reflect.Value, node *jsonpath.FieldNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { results := []reflect.Value{} nextFns := []setFieldFunc{} // If there's no input, there's no output @@ -502,7 +492,7 @@ func (j *JSONPathExtracter) evalField(input []reflect.Value, node *FieldNode, se } // evalWildcard extracts all contents of the given value -func (j *JSONPathExtracter) evalWildcard(input []reflect.Value, _ *WildcardNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalWildcard(input []reflect.Value, _ *jsonpath.WildcardNode) ([]reflect.Value, error) { results := []reflect.Value{} for _, value := range input { value, isNil := template.Indirect(value) @@ -529,7 +519,7 @@ func (j *JSONPathExtracter) evalWildcard(input []reflect.Value, _ *WildcardNode) } // evalRecursive visits the given value recursively and pushes all of them to result -func (j *JSONPathExtracter) evalRecursive(input []reflect.Value, node *RecursiveNode) ([]reflect.Value, error) { +func (j *jsonPathExtracter) evalRecursive(input []reflect.Value, node *jsonpath.RecursiveNode) ([]reflect.Value, error) { result := []reflect.Value{} for _, value := range input { results := []reflect.Value{} @@ -565,7 +555,7 @@ func (j *JSONPathExtracter) evalRecursive(input []reflect.Value, node *Recursive } // evalFilter filters array according to FilterNode -func (j *JSONPathExtracter) evalFilter(input []reflect.Value, node *FilterNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { +func (j *jsonPathExtracter) evalFilter(input []reflect.Value, node *jsonpath.FilterNode, setFn []setFieldFunc) ([]reflect.Value, []setFieldFunc, error) { results := []reflect.Value{} fns := []setFieldFunc{} for k, value := range input { diff --git a/extracter/jsonpath_test.go b/extracter/jsonpath_test.go index 0aca18b..2d4c263 100644 --- a/extracter/jsonpath_test.go +++ b/extracter/jsonpath_test.go @@ -29,10 +29,14 @@ type jsonPathTest struct { expectError bool } -func (t *jsonPathTest) Prepare(allowMissingKeys bool) (*JSONPathExtracter, error) { - jp := NewJSONPathExtracter(t.name) - jp.AllowMissingKeys(allowMissingKeys) - return jp, jp.Parse(t.template) +func (t *jsonPathTest) Prepare(allowMissingKeys bool) (Extracter, error) { + parser, err := Parse(t.template, t.template) + if err != nil { + return nil, err + } + + jp := NewJSONPathExtracter(parser, allowMissingKeys) + return jp, nil } func benchmarkJSONPath(test jsonPathTest, allowMissingKeys bool, b *testing.B) { @@ -58,6 +62,7 @@ func testJSONPath(tests []jsonPathTest, allowMissingKeys bool, t *testing.T) { t.Errorf("in %s, parse %s error %v", test.name, test.template, err) continue } + return } got, err := jp.Extract(test.input) diff --git a/extracter/merge.go b/extracter/merge.go index f9ebf0e..19e6ef9 100644 --- a/extracter/merge.go +++ b/extracter/merge.go @@ -20,10 +20,10 @@ import ( "reflect" ) -// MergeFields merges src into dst. +// mergeFields merges src into dst. // // Note: the merge operation on two nested list is replacing. -func MergeFields(dst, src map[string]interface{}) map[string]interface{} { +func mergeFields(dst, src map[string]interface{}) map[string]interface{} { for key, val := range src { if cur, ok := dst[key]; ok { if reflect.TypeOf(val) != reflect.TypeOf(cur) { @@ -34,7 +34,7 @@ func MergeFields(dst, src map[string]interface{}) map[string]interface{} { case []interface{}: dst[key] = val.([]interface{}) case map[string]interface{}: - dst[key] = MergeFields(cur, val.(map[string]interface{})) + dst[key] = mergeFields(cur, val.(map[string]interface{})) default: dst[key] = val } diff --git a/extracter/alias.go b/extracter/parser.go similarity index 56% rename from extracter/alias.go rename to extracter/parser.go index f5fafc6..4c17483 100644 --- a/extracter/alias.go +++ b/extracter/parser.go @@ -17,22 +17,27 @@ package extracter import ( + "errors" + "k8s.io/client-go/util/jsonpath" ) -type ( - Parser = jsonpath.Parser - Node = jsonpath.Node - ListNode = jsonpath.ListNode - TextNode = jsonpath.TextNode - FieldNode = jsonpath.FieldNode - ArrayNode = jsonpath.ArrayNode - FilterNode = jsonpath.FilterNode - IntNode = jsonpath.IntNode - BoolNode = jsonpath.BoolNode - FloatNode = jsonpath.FloatNode - WildcardNode = jsonpath.WildcardNode - RecursiveNode = jsonpath.RecursiveNode - UnionNode = jsonpath.UnionNode - IdentifierNode = jsonpath.IdentifierNode -) +// Parse is unlike the jsonpath.Parse, which supports multi-paths input. +// The input like `{.kind} {.apiVersion}` or +// `{range .spec.containers[*]}{.name}{end}` will result in an error. +func Parse(name, text string) (*parser, error) { + p, err := jsonpath.Parse(name, text) + if err != nil { + return nil, err + } + + if len(p.Root.Nodes) > 1 { + return nil, errors.New("not support multi-paths input") + } + + return &parser{p}, nil +} + +type parser struct { + *jsonpath.Parser +} diff --git a/extracter/parser_test.go b/extracter/parser_test.go new file mode 100644 index 0000000..ca9d68d --- /dev/null +++ b/extracter/parser_test.go @@ -0,0 +1,50 @@ +/** + * Copyright 2024 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package extracter + +import ( + "reflect" + "testing" +) + +func TestParse(t *testing.T) { + tests := []struct { + name string + text string + want *parser + wantErr bool + }{ + {name: "multi paths", text: "{.kind} {.apiVersion}", want: nil, wantErr: true}, + {name: "range path", text: "{range .spec.containers[*]}{.name}{end}", want: nil, wantErr: true}, + {name: "one path", text: "{.kind}", want: &parser{}, wantErr: false}, + {name: "empty brace", text: "{}", want: &parser{}, wantErr: false}, + {name: "empty", text: "", want: &parser{}, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Parse(tt.text, tt.text) + if (err != nil) != tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if reflect.TypeOf(got) != reflect.TypeOf(tt.want) { + t.Errorf("Parse() = %v, want %v", got, tt.want) + } + }) + } +}