From 8b944ee78c4ac4536bb1b2c75164d33206d0a6d0 Mon Sep 17 00:00:00 2001 From: "yuxuan.wang1" Date: Mon, 27 May 2024 11:35:18 +0800 Subject: [PATCH] feat: support trimming multiple IDLs --- generator/golang/backend.go | 5 +- tool/trimmer/main.go | 2 + .../config/idl_compose/idl_compose.yaml | 25 ++ .../config/trim_config/trim_config.yaml | 21 ++ .../idl_compose.yaml | 25 ++ .../trim_config.yaml | 21 ++ .../test_cases/multiple_idls/common1.thrift | 35 +++ .../test_cases/multiple_idls/common2.thrift | 35 +++ .../multiple_idls_include_common/test1.thrift | 26 +++ .../multiple_idls_include_common/test2.thrift | 26 +++ .../multiple_idls_include_common/test3.thrift | 29 +++ .../multiple_idls_include_common/test4.thrift | 29 +++ .../multiple_idls_include_common/test5.thrift | 33 +++ .../multiple_idls_include_common/test6.thrift | 31 +++ .../multiple_idls_include_common/test7.thrift | 39 ++++ .../multiple_idls_include_common/test8.thrift | 29 +++ .../test_cases/test_extend/common1.thrift | 25 ++ .../test_cases/test_extend/common2.thrift | 25 ++ .../test_cases/test_extend/common3.thrift | 28 +++ tool/trimmer/trim/config.go | 129 ++++++++++- tool/trimmer/trim/config_test.go | 132 +++++++++++ tool/trimmer/trim/idl_compose.yaml | 17 ++ tool/trimmer/trim/mark.go | 157 ++++++++----- tool/trimmer/trim/pre-process.go | 11 +- tool/trimmer/trim/traversal.go | 33 +-- tool/trimmer/trim/trimmer.go | 134 +++++------ tool/trimmer/trim/trimmer_test.go | 214 +++++++++++++++++- tool/trimmer/trim/util.go | 38 ++++ 28 files changed, 1208 insertions(+), 146 deletions(-) create mode 100644 tool/trimmer/test_cases/config/idl_compose/idl_compose.yaml create mode 100644 tool/trimmer/test_cases/config/trim_config/trim_config.yaml create mode 100644 tool/trimmer/test_cases/config/trim_config_and_idl_compose/idl_compose.yaml create mode 100644 tool/trimmer/test_cases/config/trim_config_and_idl_compose/trim_config.yaml create mode 100644 tool/trimmer/test_cases/multiple_idls/common1.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/common2.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test1.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test2.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test3.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test4.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test5.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test6.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test7.thrift create mode 100644 tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test8.thrift create mode 100644 tool/trimmer/test_cases/test_extend/common1.thrift create mode 100644 tool/trimmer/test_cases/test_extend/common2.thrift create mode 100644 tool/trimmer/test_cases/test_extend/common3.thrift create mode 100644 tool/trimmer/trim/config_test.go create mode 100644 tool/trimmer/trim/idl_compose.yaml create mode 100644 tool/trimmer/trim/util.go diff --git a/generator/golang/backend.go b/generator/golang/backend.go index caa29d2d..b8246563 100644 --- a/generator/golang/backend.go +++ b/generator/golang/backend.go @@ -89,7 +89,10 @@ func (g *GoBackend) Generate(req *plugin.Request, log backend.LogFunc) *plugin.R g.prepareUtilities() if g.utils.Features().TrimIDL { g.log.Warn("You Are Using IDL Trimmer") - tr, err := trim.TrimAST(&trim.TrimASTArg{Ast: req.AST, TrimMethods: nil, Preserve: nil}) + tr, err := trim.TrimASTWithCompose(&trim.TrimASTWithComposeArg{ + TargetAST: req.AST, + ReadCfgFromLocal: true, + }) if err != nil { g.log.Warn("trim error:", err.Error()) } diff --git a/tool/trimmer/main.go b/tool/trimmer/main.go index 8e20b224..eb2adae1 100644 --- a/tool/trimmer/main.go +++ b/tool/trimmer/main.go @@ -53,6 +53,8 @@ func main() { os.Exit(0) } + // trimmer compose + var preserveInput *bool if a.Preserve != "" { preserve, err := strconv.ParseBool(a.Preserve) diff --git a/tool/trimmer/test_cases/config/idl_compose/idl_compose.yaml b/tool/trimmer/test_cases/config/idl_compose/idl_compose.yaml new file mode 100644 index 00000000..2dd2fda4 --- /dev/null +++ b/tool/trimmer/test_cases/config/idl_compose/idl_compose.yaml @@ -0,0 +1,25 @@ +# Copyright 2024 CloudWeGo Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +idls: + test1.thrift: + trimmer: + methods: + - "TestService.func1" + - "TestService.func3" + preserve: true + preserved_structs: + - "useless" + match_go_name: true + test2.thrift: \ No newline at end of file diff --git a/tool/trimmer/test_cases/config/trim_config/trim_config.yaml b/tool/trimmer/test_cases/config/trim_config/trim_config.yaml new file mode 100644 index 00000000..9f975f5f --- /dev/null +++ b/tool/trimmer/test_cases/config/trim_config/trim_config.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 CloudWeGo Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +methods: + - "TestService.func1" + - "TestService.func3" +preserve: true +preserved_structs: + - "useless" +match_go_name: true \ No newline at end of file diff --git a/tool/trimmer/test_cases/config/trim_config_and_idl_compose/idl_compose.yaml b/tool/trimmer/test_cases/config/trim_config_and_idl_compose/idl_compose.yaml new file mode 100644 index 00000000..2dd2fda4 --- /dev/null +++ b/tool/trimmer/test_cases/config/trim_config_and_idl_compose/idl_compose.yaml @@ -0,0 +1,25 @@ +# Copyright 2024 CloudWeGo Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +idls: + test1.thrift: + trimmer: + methods: + - "TestService.func1" + - "TestService.func3" + preserve: true + preserved_structs: + - "useless" + match_go_name: true + test2.thrift: \ No newline at end of file diff --git a/tool/trimmer/test_cases/config/trim_config_and_idl_compose/trim_config.yaml b/tool/trimmer/test_cases/config/trim_config_and_idl_compose/trim_config.yaml new file mode 100644 index 00000000..9f975f5f --- /dev/null +++ b/tool/trimmer/test_cases/config/trim_config_and_idl_compose/trim_config.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 CloudWeGo Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +methods: + - "TestService.func1" + - "TestService.func3" +preserve: true +preserved_structs: + - "useless" +match_go_name: true \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/common1.thrift b/tool/trimmer/test_cases/multiple_idls/common1.thrift new file mode 100644 index 00000000..04e4ce29 --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/common1.thrift @@ -0,0 +1,35 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go tests.multiple.common + +enum Common1Enum1 { + ONE + TWO + THREE +} + +enum Common1Enum2 { + ONE + TWO + THERR +} + +struct Common1Struct1 { + 1: required string field +} + +struct Common1Struct2 { + 1: required string field +} diff --git a/tool/trimmer/test_cases/multiple_idls/common2.thrift b/tool/trimmer/test_cases/multiple_idls/common2.thrift new file mode 100644 index 00000000..a27ed62a --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/common2.thrift @@ -0,0 +1,35 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go tests.multiple.common + +enum Common2Enum1 { + ONE + TWO + THREE +} + +enum Common2Enum2 { + ONE + TWO + THERR +} + +struct Common2Struct1 { + 1: required string field +} + +struct Common2Struct2 { + 1: required string field +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test1.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test1.thrift new file mode 100644 index 00000000..44e20603 --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test1.thrift @@ -0,0 +1,26 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" + +struct Test1Struct1 { + 1: required common1.Common1Enum1 enumField + 2: required common1.Common1Struct1 structField +} + +service Test1 { + string Process(1: Test1Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test2.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test2.thrift new file mode 100644 index 00000000..8e5af4ae --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test2.thrift @@ -0,0 +1,26 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" + +struct Test2Struct1 { + 1: required common1.Common1Enum2 enumField + 2: required common1.Common1Struct2 structField +} + +service Test2 { + string Process(1: Test2Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test3.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test3.thrift new file mode 100644 index 00000000..41b84b55 --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test3.thrift @@ -0,0 +1,29 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" +include "../common2.thrift" + +struct Test3Struct1 { + 1: required common1.Common1Enum1 enumField1 + 2: required common1.Common1Struct1 structField1 + 3: required common2.Common2Enum1 enumField2 + 4: required common2.Common2Struct1 structField2 +} + +service Test3 { + string Process(1: Test3Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test4.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test4.thrift new file mode 100644 index 00000000..e07393ca --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test4.thrift @@ -0,0 +1,29 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" +include "../common2.thrift" + +struct Test4Struct1 { + 1: required common1.Common1Enum2 enumField1 + 2: required common1.Common1Struct2 structField1 + 3: required common2.Common2Enum2 enumField2 + 4: required common2.Common2Struct2 structField2 +} + +service Test4 { + string Process(1: Test4Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test5.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test5.thrift new file mode 100644 index 00000000..214b776d --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test5.thrift @@ -0,0 +1,33 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" +include "../common2.thrift" + +struct Test5Struct1 { + 1: required common1.Common1Enum1 enumField1 + 2: required common1.Common1Struct1 structField1 + 3: required common2.Common2Enum1 enumField2 + 4: required common2.Common2Struct1 structField2 +} + +struct Test5Struct2 { + 1: required string stringField +} + +service Test5 { + string Process(1: Test5Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test6.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test6.thrift new file mode 100644 index 00000000..763c8975 --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test6.thrift @@ -0,0 +1,31 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" +include "../common2.thrift" +include "test5.thrift" + +struct Test6Struct1 { + 1: required common1.Common1Enum2 enumField1 + 2: required common1.Common1Struct2 structField1 + 3: required common2.Common2Enum2 enumField2 + 4: required common2.Common2Struct2 structField2 +} + +service Test6 { + string Process(1: Test6Struct1 req) + string ProcessAnotherIDL(1: test5.Test5Struct2 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test7.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test7.thrift new file mode 100644 index 00000000..b4f1b56e --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test7.thrift @@ -0,0 +1,39 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" +include "../common2.thrift" + +struct Test7Struct1 { + 1: required common1.Common1Enum1 enumField1 + 2: required common1.Common1Struct1 structField1 + 3: required common2.Common2Enum1 enumField2 + 4: required common2.Common2Struct1 structField2 +} + +// @preserve +struct Test7Struct2 { + 1: required string stringField +} + +struct Test7Struct3 { + 1: required string stringField +} + +service Test7 { + string Process(1: Test7Struct1 req) + string Echo(1: string req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test8.thrift b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test8.thrift new file mode 100644 index 00000000..d58127f0 --- /dev/null +++ b/tool/trimmer/test_cases/multiple_idls/multiple_idls_include_common/test8.thrift @@ -0,0 +1,29 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go multiple.include.common + +include "../common1.thrift" +include "../common2.thrift" + +struct Test8Struct1 { + 1: required common1.Common1Enum2 enumField1 + 2: required common1.Common1Struct2 structField1 + 3: required common2.Common2Enum2 enumField2 + 4: required common2.Common2Struct2 structField2 +} + +service Test8 { + string Process(1: Test8Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/test_extend/common1.thrift b/tool/trimmer/test_cases/test_extend/common1.thrift new file mode 100644 index 00000000..d973fedd --- /dev/null +++ b/tool/trimmer/test_cases/test_extend/common1.thrift @@ -0,0 +1,25 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go tests.extend.common + +include "common2.thrift" + +struct Common1Struct1 { + 1: required string stringField +} + +service Common1 extends common2.Common2 { + string ProcessCommon1(1: Common1Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/test_extend/common2.thrift b/tool/trimmer/test_cases/test_extend/common2.thrift new file mode 100644 index 00000000..b5829e64 --- /dev/null +++ b/tool/trimmer/test_cases/test_extend/common2.thrift @@ -0,0 +1,25 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go tests.extend.common + +include "common3.thrift" + +struct Common2Struct1 { + 1: required string stringField +} + +service Common2 extends common3.Common3 { + string ProcessCommon2(1: Common2Struct1 req) +} \ No newline at end of file diff --git a/tool/trimmer/test_cases/test_extend/common3.thrift b/tool/trimmer/test_cases/test_extend/common3.thrift new file mode 100644 index 00000000..6d795050 --- /dev/null +++ b/tool/trimmer/test_cases/test_extend/common3.thrift @@ -0,0 +1,28 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace go tests.extend.common + +struct Common3Struct1 { + 1: required string stringField +} + +struct Commmon3Struct2 { + 1: required string stringField +} + +service Common3 { + string ProcessCommon3(1: Common3Struct1 req) + string Echo(1: Commmon3Struct2 req) +} \ No newline at end of file diff --git a/tool/trimmer/trim/config.go b/tool/trimmer/trim/config.go index a88a4977..bbc02bf0 100644 --- a/tool/trimmer/trim/config.go +++ b/tool/trimmer/trim/config.go @@ -22,7 +22,51 @@ import ( "gopkg.in/yaml.v3" ) -var DefaultYamlFileName = "trim_config.yaml" +var ( + DefaultYamlFileName = "trim_config.yaml" + DefaultIDLComposeFileName = "idl_compose.yaml" +) + +// IDLArguments contains all arguments about the IDL. +// For now, it only contains YamlArguments. +type IDLArguments struct { + Trimmer *YamlArguments `yaml:"trimmer,omitempty"` +} + +func (args *IDLArguments) setDefault() { + if args == nil { + return + } + if args.Trimmer != nil { + args.Trimmer.setDefault() + } else { + trimArgs := &YamlArguments{} + trimArgs.setDefault() + args.Trimmer = trimArgs + } +} + +// IDLComposeArguments contains all IDLs and their arguments. +type IDLComposeArguments struct { + // path is the path of IDL based on working directory + // e.g. "idl/sample.thrift" + IDLs map[string]*IDLArguments `yaml:"idls,omitempty"` +} + +func (args *IDLComposeArguments) setDefault() { + if args == nil { + return + } + for filename, idlArgs := range args.IDLs { + if idlArgs != nil { + idlArgs.setDefault() + } else { + newIDLArgs := &IDLArguments{} + newIDLArgs.setDefault() + args.IDLs[filename] = newIDLArgs + } + } +} type YamlArguments struct { Methods []string `yaml:"methods,omitempty"` @@ -31,6 +75,20 @@ type YamlArguments struct { MatchGoName *bool `yaml:"match_go_name,omitempty"` } +func (arg *YamlArguments) setDefault() { + if arg == nil { + return + } + if arg.Preserve == nil { + t := true + arg.Preserve = &t + } + if arg.MatchGoName == nil { + t := false + arg.MatchGoName = &t + } +} + func ParseYamlConfig(path string) *YamlArguments { cfg := YamlArguments{} dataBytes, err := ioutil.ReadFile(filepath.Join(path, DefaultYamlFileName)) @@ -43,13 +101,70 @@ func ParseYamlConfig(path string) *YamlArguments { fmt.Println("unmarshal yaml config fail:", err) return nil } - if cfg.Preserve == nil { - t := true - cfg.Preserve = &t + cfg.setDefault() + return &cfg +} + +func ParseIDLComposeConfig(dir string) *IDLComposeArguments { + cfg := IDLComposeArguments{} + path := filepath.Join(dir, DefaultIDLComposeFileName) + data, err := ioutil.ReadFile(path) + if err != nil { + return nil } - if cfg.MatchGoName == nil { - t := false - cfg.MatchGoName = &t + fmt.Println("using idl_compose config:", path) + err = yaml.Unmarshal(data, &cfg) + if err != nil { + fmt.Println("unmarshal idl_compose config fail:", err) + return nil + } + // set default value + for filename, idl := range cfg.IDLs { + if idl == nil { + newIdl := &IDLArguments{} + cfg.IDLs[filename] = newIdl + idl = newIdl + } + if idl.Trimmer == nil { + idl.Trimmer = &YamlArguments{} + } + idl.Trimmer.setDefault() } + return &cfg } + +func newIDLComposeArgumentsWithTargetAST(filename string) *IDLComposeArguments { + trimCfg := &YamlArguments{} + trimCfg.setDefault() + return &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + filename: { + Trimmer: trimCfg, + }, + }, + } +} + +func extractIDLComposeConfigFromDir(dir string, targetAST string) *IDLComposeArguments { + var cfg *IDLComposeArguments + cfg = ParseIDLComposeConfig(dir) + // idl_compose.yaml has higher priority + if cfg != nil { + return cfg + } + // if there is no idl_compose.yaml, use trim_config.yaml + trimCfg := ParseYamlConfig(dir) + if trimCfg != nil { + cfg = &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + targetAST: { + Trimmer: trimCfg, + }, + }, + } + return cfg + } + + return nil +} diff --git a/tool/trimmer/trim/config_test.go b/tool/trimmer/trim/config_test.go new file mode 100644 index 00000000..dd65d71a --- /dev/null +++ b/tool/trimmer/trim/config_test.go @@ -0,0 +1,132 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trim + +import ( + "github.com/cloudwego/thriftgo/pkg/test" + "testing" +) + +func Test_extractIDLComposeConfigFromDir(t *testing.T) { + testcases := []struct { + desc string + dir string + targetAST string + expect func(t *testing.T, cfg *IDLComposeArguments) + }{ + { + desc: "only trim_config.yaml", + dir: "../test_cases/config/trim_config", + targetAST: "test.thrift", + expect: func(t *testing.T, cfg *IDLComposeArguments) { + preserve := true + matchGoName := true + expect := &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + "test.thrift": { + Trimmer: &YamlArguments{ + Methods: []string{ + "TestService.func1", + "TestService.func3", + }, + Preserve: &preserve, + PreservedStructs: []string{ + "useless", + }, + MatchGoName: &matchGoName, + }, + }, + }, + } + test.DeepEqual(t, cfg, expect) + }, + }, + { + desc: "only idl_compose.yaml", + dir: "../test_cases/config/idl_compose", + targetAST: "test1.thrift", + expect: func(t *testing.T, cfg *IDLComposeArguments) { + preserve := true + matchGoNameTrue := true + matchGoNameFalse := false + expect := &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + "test1.thrift": { + Trimmer: &YamlArguments{ + Methods: []string{ + "TestService.func1", + "TestService.func3", + }, + Preserve: &preserve, + PreservedStructs: []string{ + "useless", + }, + MatchGoName: &matchGoNameTrue, + }, + }, + "test2.thrift": { + Trimmer: &YamlArguments{ + Preserve: &preserve, + MatchGoName: &matchGoNameFalse, + }, + }, + }, + } + test.DeepEqual(t, cfg, expect) + }, + }, + { + desc: "both trim_config.yaml and idl_compose.yaml, only idl_compose.yaml will be used", + dir: "../test_cases/config/trim_config_and_idl_compose", + targetAST: "test1.thrift", + expect: func(t *testing.T, cfg *IDLComposeArguments) { + preserve := true + matchGoNameTrue := true + matchGoNameFalse := false + expect := &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + "test1.thrift": { + Trimmer: &YamlArguments{ + Methods: []string{ + "TestService.func1", + "TestService.func3", + }, + Preserve: &preserve, + PreservedStructs: []string{ + "useless", + }, + MatchGoName: &matchGoNameTrue, + }, + }, + "test2.thrift": { + Trimmer: &YamlArguments{ + Preserve: &preserve, + MatchGoName: &matchGoNameFalse, + }, + }, + }, + } + test.DeepEqual(t, cfg, expect) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + cfg := extractIDLComposeConfigFromDir(tc.dir, tc.targetAST) + tc.expect(t, cfg) + }) + } +} diff --git a/tool/trimmer/trim/idl_compose.yaml b/tool/trimmer/trim/idl_compose.yaml new file mode 100644 index 00000000..cef5a98f --- /dev/null +++ b/tool/trimmer/trim/idl_compose.yaml @@ -0,0 +1,17 @@ +# Copyright 2024 CloudWeGo Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +idls: + ../test_cases/multiple_idls/multiple_idls_include_common/test1.thrift: + ../test_cases/multiple_idls/multiple_idls_include_common/test2.thrift: \ No newline at end of file diff --git a/tool/trimmer/trim/mark.go b/tool/trimmer/trim/mark.go index e7d60ada..85322406 100644 --- a/tool/trimmer/trim/mark.go +++ b/tool/trimmer/trim/mark.go @@ -17,19 +17,59 @@ package trim import ( "strings" + "github.com/dlclark/regexp2" + "github.com/cloudwego/thriftgo/parser" ) +const ( + includePrefix = "trimmer_include_prefix_" + functionPrefix = "trimmer_function_prefix_" +) + +// refresh initialize the trimMethods, preservedStructs and trimmed statistical data. +// this function must be called before markAST +func (t *Trimmer) refresh(ast *parser.Thrift, arg *YamlArguments) { + if arg == nil { + return + } + var err error + // deal with trimMethods + t.trimMethods = make([]*regexp2.Regexp, len(arg.Methods)) + t.trimMethodValid = make([]bool, len(arg.Methods)) + t.matchGoName = *arg.MatchGoName + for i, method := range arg.Methods { + parts := strings.Split(method, ".") + if len(parts) < 2 { + if len(ast.Services) == 1 { + arg.Methods[i] = ast.Services[0].Name + "." + method + } else { + arg.Methods[i] = ast.Services[len(ast.Services)-1].Name + "." + method + // println("please specify service name!\n -m usage: -m [service_name.method_name]") + // os.Exit(2) + } + } + t.trimMethods[i], err = regexp2.Compile(arg.Methods[i], 0) + check(err) + } + // deal with preservedStructs + t.preservedStructs = arg.PreservedStructs + t.forceTrimming = false + if len(t.preservedStructs) <= 0 { + t.forceTrimming = true + } +} + // mark the used part of ast -func (t *Trimmer) markAST(ast *parser.Thrift) { - t.marks[ast.Filename] = make(map[interface{}]bool) - t.preProcess(ast, ast.Filename) +func (t *Trimmer) markAST(ast *parser.Thrift, arg *YamlArguments) { + t.refresh(ast, arg) + t.preProcess(ast) for _, service := range ast.Services { - t.markService(service, ast, ast.Filename) + t.markService(service, ast) } t.cleanServiceExtends() - t.markKeptPart(ast, ast.Filename) + t.markKeptPart(ast) } func toGoName(input string) string { @@ -44,13 +84,14 @@ func toGoName(input string) string { return result.String() } -func (t *Trimmer) markService(svc *parser.Service, ast *parser.Thrift, filename string) { - if t.marks[filename][svc] { +func (t *Trimmer) markService(svc *parser.Service, ast *parser.Thrift) { + filename := ast.Filename + if t.marks[filename][svc.Name] { return } if len(t.trimMethods) == 0 { - t.marks[filename][svc] = true + t.marks[filename][svc.Name] = true } for _, function := range svc.Functions { @@ -62,29 +103,29 @@ func (t *Trimmer) markService(svc *parser.Service, ast *parser.Thrift, filename } if ok, _ := method.MatchString(funcName); ok { if funcName == method.String() || !strings.HasPrefix(funcName, method.String()) { - t.marks[filename][svc] = true - t.markFunction(function, ast, filename) + t.marks[filename][svc.Name] = true + t.markFunction(function, ast, svc.Name) t.trimMethodValid[i] = true } } } continue } - t.markFunction(function, ast, filename) + t.markFunction(function, ast, svc.Name) } if len(t.trimMethods) != 0 && (svc.Extends != "" || svc.Reference != nil) { - t.traceExtendMethod(svc, svc, ast, filename) + t.traceExtendMethod(svc, svc, ast) } - if svc.Extends != "" && t.marks[filename][svc] { + if svc.Extends != "" && t.marks[filename][svc.Name] { // handle extension if svc.Reference != nil { theInclude := ast.Includes[svc.Reference.Index] t.markInclude(ast.Includes[svc.Reference.Index], filename) for _, service := range theInclude.Reference.Services { if service.Name == svc.Reference.Name { - t.markService(service, theInclude.Reference, filename) + t.markService(service, theInclude.Reference) break } } @@ -92,98 +133,102 @@ func (t *Trimmer) markService(svc *parser.Service, ast *parser.Thrift, filename } } -func (t *Trimmer) markFunction(function *parser.Function, ast *parser.Thrift, filename string) { - t.marks[filename][function] = true +func (t *Trimmer) markFunction(function *parser.Function, ast *parser.Thrift, serviceName string) { + t.marks[ast.Filename][functionIdentifier(serviceName, function.Name)] = true for _, arg := range function.Arguments { - t.markType(arg.Type, ast, filename) + t.markType(arg.Type, ast) } for _, throw := range function.Throws { - t.markType(throw.Type, ast, filename) + t.markType(throw.Type, ast) } if !function.Void { - t.markType(function.FunctionType, ast, filename) + t.markType(function.FunctionType, ast) } } -func (t *Trimmer) markType(theType *parser.Type, ast *parser.Thrift, filename string) { +func (t *Trimmer) markType(theType *parser.Type, ast *parser.Thrift) { + filename := ast.Filename // plain type if theType.Category <= 8 && theType.IsTypedef == nil { return } if theType.KeyType != nil { - t.markType(theType.KeyType, ast, filename) + t.markType(theType.KeyType, ast) } if theType.ValueType != nil { - t.markType(theType.ValueType, ast, filename) + t.markType(theType.ValueType, ast) } baseAST := ast if theType.Reference != nil { // if referenced, redirect to included ast baseAST = ast.Includes[theType.Reference.Index].Reference + if _, ok := t.marks[baseAST.Filename]; !ok { + t.marks[baseAST.Filename] = make(map[string]bool) + } t.markInclude(ast.Includes[theType.Reference.Index], filename) } if theType.IsTypedef != nil { - t.markTypeDef(theType, baseAST, filename) + t.markTypeDef(theType, baseAST) return } if theType.Category.IsStruct() { for _, str := range baseAST.Structs { if str.Name == theType.Name || (theType.Reference != nil && str.Name == theType.Reference.Name) { - t.markStructLike(str, baseAST, filename) + t.markStructLike(str, baseAST) break } } } else if theType.Category.IsException() { for _, str := range baseAST.Exceptions { if str.Name == theType.Name || (theType.Reference != nil && str.Name == theType.Reference.Name) { - t.markStructLike(str, baseAST, filename) + t.markStructLike(str, baseAST) break } } } else if theType.Category.IsUnion() { for _, str := range baseAST.Unions { if str.Name == theType.Name || (theType.Reference != nil && str.Name == theType.Reference.Name) { - t.markStructLike(str, baseAST, filename) + t.markStructLike(str, baseAST) break } } } else if theType.Category.IsEnum() { for _, enum := range baseAST.Enums { if enum.Name == theType.Name || (theType.Reference != nil && enum.Name == theType.Reference.Name) { - t.markEnum(enum, filename) + t.markEnum(enum, baseAST) break } } } } -func (t *Trimmer) markStructLike(str *parser.StructLike, ast *parser.Thrift, filename string) { - if t.marks[filename][str] { +func (t *Trimmer) markStructLike(str *parser.StructLike, ast *parser.Thrift) { + if t.marks[ast.Filename][str.Name] { return } - t.marks[filename][str] = true + t.marks[ast.Filename][str.Name] = true for _, field := range str.Fields { - t.markType(field.Type, ast, filename) + t.markType(field.Type, ast) } } -func (t *Trimmer) markEnum(enum *parser.Enum, filename string) { - t.marks[filename][enum] = true +func (t *Trimmer) markEnum(enum *parser.Enum, ast *parser.Thrift) { + t.marks[ast.Filename][enum.Name] = true } -func (t *Trimmer) markTypeDef(theType *parser.Type, ast *parser.Thrift, filename string) { +func (t *Trimmer) markTypeDef(theType *parser.Type, ast *parser.Thrift) { if theType.IsTypedef == nil { return } for i, typedef := range ast.Typedefs { if typedef.Alias == theType.Name { - if !t.marks[filename][ast.Typedefs[i]] { - t.marks[filename][ast.Typedefs[i]] = true - t.markType(typedef.Type, ast, filename) + if !t.marks[ast.Filename][ast.Typedefs[i].Alias] { + t.marks[ast.Filename][ast.Typedefs[i].Alias] = true + t.markType(typedef.Type, ast) } return } @@ -192,10 +237,10 @@ func (t *Trimmer) markTypeDef(theType *parser.Type, ast *parser.Thrift, filename func (t *Trimmer) markInclude(include *parser.Include, filename string) { include.Reference.Name2Category = nil - if t.marks[filename][include] { + if t.marks[filename][includePrefix+include.Path] { return } - t.marks[filename][include] = true + t.marks[filename][includePrefix+include.Path] = true // t.markKeptPart(include.Reference, filename) } @@ -214,36 +259,37 @@ func (t *Trimmer) cleanServiceExtends() { } } -func (t *Trimmer) markKeptPart(ast *parser.Thrift, filename string) bool { +func (t *Trimmer) markKeptPart(ast *parser.Thrift) bool { + filename := ast.Filename ret := false for _, constant := range ast.Constants { - t.markType(constant.Type, ast, filename) + t.markType(constant.Type, ast) ret = true } for _, typedef := range ast.Typedefs { - t.markType(typedef.Type, ast, filename) + t.markType(typedef.Type, ast) ret = true } if !t.forceTrimming { for _, str := range ast.Structs { - if !t.marks[filename][str] && t.checkPreserve(str) { - t.markStructLike(str, ast, filename) + if !t.marks[filename][str.Name] && t.checkPreserve(str) { + t.markStructLike(str, ast) ret = true } } for _, str := range ast.Unions { - if !t.marks[filename][str] && t.checkPreserve(str) { - t.markStructLike(str, ast, filename) + if !t.marks[filename][str.Name] && t.checkPreserve(str) { + t.markStructLike(str, ast) ret = true } } for _, str := range ast.Exceptions { - if !t.marks[filename][str] && t.checkPreserve(str) { - t.markStructLike(str, ast, filename) + if !t.marks[filename][str.Name] && t.checkPreserve(str) { + t.markStructLike(str, ast) ret = true } } @@ -252,13 +298,14 @@ func (t *Trimmer) markKeptPart(ast *parser.Thrift, filename string) bool { } // for -m, trace the extends and find specified method to base on -func (t *Trimmer) traceExtendMethod(father, svc *parser.Service, ast *parser.Thrift, filename string) (ret bool) { +func (t *Trimmer) traceExtendMethod(father, svc *parser.Service, ast *parser.Thrift) (ret bool) { + filename := ast.Filename for _, function := range svc.Functions { funcName := father.Name + "." + function.Name for i, method := range t.trimMethods { if ok, _ := method.MatchString(funcName); ok { - t.marks[filename][svc] = true - t.markFunction(function, ast, filename) + t.marks[filename][svc.Name] = true + t.markFunction(function, ast, svc.Name) t.trimMethodValid[i] = true ret = true } @@ -284,14 +331,14 @@ func (t *Trimmer) traceExtendMethod(father, svc *parser.Service, ast *parser.Thr } } } - back := t.traceExtendMethod(father, nextSvc, nextAst, filename) + back := t.traceExtendMethod(father, nextSvc, nextAst) if !back { t.markServiceExtends(svc) } ret = back || ret } if ret { - t.marks[filename][svc] = true + t.marks[filename][svc.Name] = true if svc.Reference != nil { t.markInclude(ast.Includes[svc.Reference.Index], filename) } @@ -311,3 +358,7 @@ func (t *Trimmer) checkPreserve(theStruct *parser.StructLike) bool { } return t.preserveRegex.MatchString(strings.ToLower(theStruct.ReservedComments)) } + +func functionIdentifier(svcName, funcName string) string { + return functionPrefix + svcName + "." + funcName +} diff --git a/tool/trimmer/trim/pre-process.go b/tool/trimmer/trim/pre-process.go index 27134990..c3cad758 100644 --- a/tool/trimmer/trim/pre-process.go +++ b/tool/trimmer/trim/pre-process.go @@ -16,12 +16,15 @@ package trim import "github.com/cloudwego/thriftgo/parser" -func (t *Trimmer) preProcess(ast *parser.Thrift, filename string) bool { - ret := t.markKeptPart(ast, filename) +func (t *Trimmer) preProcess(ast *parser.Thrift) bool { + if _, ok := t.marks[ast.Filename]; !ok { + t.marks[ast.Filename] = make(map[string]bool) + } + ret := t.markKeptPart(ast) for i, include := range ast.Includes { - marked := t.preProcess(include.Reference, filename) + marked := t.preProcess(include.Reference) if marked { - t.marks[filename][ast.Includes[i]] = true + t.marks[ast.Filename][includePrefix+ast.Includes[i].Path] = true ret = true } } diff --git a/tool/trimmer/trim/traversal.go b/tool/trimmer/trim/traversal.go index ec6823f8..d4706b0d 100644 --- a/tool/trimmer/trim/traversal.go +++ b/tool/trimmer/trim/traversal.go @@ -19,12 +19,19 @@ import ( ) // traverse and remove the unmarked part of ast -func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { +func (t *Trimmer) traversal(ast *parser.Thrift) { + t.countStructs(ast) + t.doTraversal(ast) +} + +func (t *Trimmer) doTraversal(ast *parser.Thrift) { + // deal with trimmed statistical data + filename := ast.Filename var listInclude []*parser.Include for i := range ast.Includes { - if t.marks[filename][ast.Includes[i]] || len(ast.Includes[i].Reference.Constants)+ + if t.marks[filename][includePrefix+ast.Includes[i].Path] || len(ast.Includes[i].Reference.Constants)+ len(ast.Includes[i].Reference.Enums)+len(ast.Includes[i].Reference.Typedefs) > 0 { - t.traversal(ast.Includes[i].Reference, filename) + t.doTraversal(ast.Includes[i].Reference) listInclude = append(listInclude, ast.Includes[i]) } } @@ -32,7 +39,7 @@ func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { var listStruct []*parser.StructLike for i := range ast.Structs { - if t.marks[filename][ast.Structs[i]] || t.checkPreserve(ast.Structs[i]) { + if t.marks[filename][ast.Structs[i].Name] || t.checkPreserve(ast.Structs[i]) { listStruct = append(listStruct, ast.Structs[i]) t.fieldsTrimmed -= len(ast.Structs[i].Fields) } @@ -41,7 +48,7 @@ func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { var listUnion []*parser.StructLike for i := range ast.Unions { - if t.marks[filename][ast.Unions[i]] || t.checkPreserve(ast.Unions[i]) { + if t.marks[filename][ast.Unions[i].Name] || t.checkPreserve(ast.Unions[i]) { listUnion = append(listUnion, ast.Unions[i]) t.fieldsTrimmed -= len(ast.Unions[i].Fields) } @@ -50,7 +57,7 @@ func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { var listException []*parser.StructLike for i := range ast.Exceptions { - if t.marks[filename][ast.Exceptions[i]] || t.checkPreserve(ast.Exceptions[i]) { + if t.marks[filename][ast.Exceptions[i].Name] || t.checkPreserve(ast.Exceptions[i]) { listException = append(listException, ast.Exceptions[i]) t.fieldsTrimmed -= len(ast.Exceptions[i].Fields) } @@ -59,16 +66,14 @@ func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { var listService []*parser.Service for i := range ast.Services { - if t.marks[filename][ast.Services[i]] { - if len(t.trimMethods) != 0 { - var trimmedMethods []*parser.Function - for j := range ast.Services[i].Functions { - if t.marks[filename][ast.Services[i].Functions[j]] { - trimmedMethods = append(trimmedMethods, ast.Services[i].Functions[j]) - } + if t.marks[filename][ast.Services[i].Name] { + var trimmedMethods []*parser.Function + for j := range ast.Services[i].Functions { + if t.marks[filename][functionIdentifier(ast.Services[i].Name, ast.Services[i].Functions[j].Name)] { + trimmedMethods = append(trimmedMethods, ast.Services[i].Functions[j]) } - ast.Services[i].Functions = trimmedMethods } + ast.Services[i].Functions = trimmedMethods listService = append(listService, ast.Services[i]) t.fieldsTrimmed -= len(ast.Services[i].Functions) } diff --git a/tool/trimmer/trim/trimmer.go b/tool/trimmer/trim/trimmer.go index c774471d..c22be2ac 100644 --- a/tool/trimmer/trim/trimmer.go +++ b/tool/trimmer/trim/trimmer.go @@ -15,17 +15,15 @@ package trim import ( - "fmt" + "errors" "os" "regexp" - "strings" "github.com/cloudwego/thriftgo/utils/dir_utils" "github.com/dlclark/regexp2" "github.com/cloudwego/thriftgo/parser" - "github.com/cloudwego/thriftgo/semantic" ) type Trimmer struct { @@ -33,7 +31,8 @@ type Trimmer struct { // ast of the file asts map[string]*parser.Thrift // mark the parts of the file's ast that is used - marks map[string]map[interface{}]bool + // key is IDL filename + marks map[string]map[string]bool outDir string // use -m trimMethods []*regexp2.Regexp @@ -77,6 +76,12 @@ func (t *TrimResultInfo) FieldTrimmedPercentage() float64 { return float64(t.FieldsTrimmed) / float64(t.FieldsTotal) * 100 } +type TrimASTWithComposeArg struct { + Cfg *IDLComposeArguments + TargetAST *parser.Thrift + ReadCfgFromLocal bool +} + // TrimAST parse the cfg and trim the single AST func TrimAST(arg *TrimASTArg) (trimResultInfo *TrimResultInfo, err error) { var preservedStructs []string @@ -100,72 +105,76 @@ func TrimAST(arg *TrimASTArg) (trimResultInfo *TrimResultInfo, err error) { if arg.Preserve != nil { forceTrim = !*arg.Preserve } + preserve := !forceTrim matchGoName := false if arg.MatchGoName != nil { matchGoName = *arg.MatchGoName } - return doTrimAST(arg.Ast, arg.TrimMethods, forceTrim, matchGoName, preservedStructs) + return TrimASTWithCompose(&TrimASTWithComposeArg{ + Cfg: &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + arg.Ast.Filename: { + Trimmer: &YamlArguments{ + Methods: arg.TrimMethods, + Preserve: &preserve, + PreservedStructs: preservedStructs, + MatchGoName: &matchGoName, + }, + }, + }, + }, + TargetAST: arg.Ast, + }) } -// doTrimAST trim the single AST, pass method names if -m specified -func doTrimAST(ast *parser.Thrift, trimMethods []string, forceTrimming bool, matchGoName bool, preservedStructs []string) ( - trimResultInfo *TrimResultInfo, err error) { - trimmer, err := newTrimmer(nil, "") - if err != nil { - return nil, err +func TrimASTWithCompose(arg *TrimASTWithComposeArg) (trimResultInfo *TrimResultInfo, err error) { + if arg == nil { + return nil, errors.New("TrimASTWithComposeArg is nil") } - trimmer.asts[ast.Filename] = ast - trimmer.trimMethods = make([]*regexp2.Regexp, len(trimMethods)) - trimmer.trimMethodValid = make([]bool, len(trimMethods)) - trimmer.forceTrimming = forceTrimming - trimmer.matchGoName = matchGoName - for i, method := range trimMethods { - parts := strings.Split(method, ".") - if len(parts) < 2 { - if len(ast.Services) == 1 { - trimMethods[i] = ast.Services[0].Name + "." + method - } else { - trimMethods[i] = ast.Services[len(ast.Services)-1].Name + "." + method - // println("please specify service name!\n -m usage: -m [service_name.method_name]") - // os.Exit(2) - - } - } - trimmer.trimMethods[i], err = regexp2.Compile(trimMethods[i], 0) - if err != nil { - return nil, err + if arg.TargetAST == nil { + return nil, errors.New("TrimASTWithComposeArg.TargetAST is nil") + } + cfg := arg.Cfg + // When ReadCfgFromLocal is set, local cfg has higher priority and the passed cfg would be ignored + if arg.ReadCfgFromLocal { + wd, err := dir_utils.Getwd() + if err == nil { + cfg = extractIDLComposeConfigFromDir(wd, arg.TargetAST.Filename) } } - trimmer.preservedStructs = preservedStructs - trimmer.countStructs(ast) - originStructsNum := trimmer.structsTrimmed - originFieldNum := trimmer.fieldsTrimmed - trimmer.markAST(ast) - trimmer.traversal(ast, ast.Filename) - if path := parser.CircleDetect(ast); len(path) > 0 { - return nil, fmt.Errorf("found include circle:\n\t%s", path) - } - checker := semantic.NewChecker(semantic.Options{FixWarnings: true}) - _, err = checker.CheckAll(ast) - if err != nil { - return nil, err + if cfg == nil { + cfg = newIDLComposeArgumentsWithTargetAST(arg.TargetAST.Filename) } - err = semantic.ResolveSymbols(ast) + cfg.setDefault() + + trimmer, err := newTrimmer(nil, "") if err != nil { return nil, err } - for i, method := range trimMethods { - if !trimmer.trimMethodValid[i] { - return nil, fmt.Errorf("err: method %s not found!\n", method) + var originStructsNum, originFieldsNum int + var structsTrimmed, fieldsTrimmed int + for path, idlArg := range cfg.IDLs { + var ast *parser.Thrift + if arg.TargetAST.Filename == path { + continue } + ast = parseAndCheckAST(path, nil, true) + trimmer.markAST(ast, idlArg.Trimmer) } + trimmer.markAST(arg.TargetAST, cfg.IDLs[arg.TargetAST.Filename].Trimmer) + // trimmer.marks now have the complete context, we can traverse the target AST + trimmer.countStructs(arg.TargetAST) + originStructsNum, originFieldsNum = trimmer.structsTrimmed, trimmer.fieldsTrimmed + trimmer.doTraversal(arg.TargetAST) + structsTrimmed, fieldsTrimmed = trimmer.structsTrimmed, trimmer.fieldsTrimmed + checkAST(arg.TargetAST) return &TrimResultInfo{ - StructsTrimmed: trimmer.structsTrimmed, - FieldsTrimmed: trimmer.fieldsTrimmed, + StructsTrimmed: structsTrimmed, + FieldsTrimmed: fieldsTrimmed, StructsTotal: originStructsNum, - FieldsTotal: originFieldNum, + FieldsTotal: originFieldsNum, }, nil } @@ -178,17 +187,9 @@ func Trim(files, includeDir []string, outDir string) error { for _, filename := range files { // go through parse process - ast, err := parser.ParseFile(filename, includeDir, true) - check(err) - if path := parser.CircleDetect(ast); len(path) > 0 { - check(fmt.Errorf("found include circle:\n\t%s", path)) - } - checker := semantic.NewChecker(semantic.Options{FixWarnings: true}) - _, err = checker.CheckAll(ast) - check(err) - check(semantic.ResolveSymbols(ast)) + ast := parseAndCheckAST(filename, includeDir, true) trimmer.asts[filename] = ast - trimmer.markAST(ast) + trimmer.markAST(ast, nil) // TODO: handle multi files and dump to 'xxx.thrift' } @@ -196,6 +197,13 @@ func Trim(files, includeDir []string, outDir string) error { } func (t *Trimmer) countStructs(ast *parser.Thrift) { + // refresh + t.fieldsTrimmed = 0 + t.structsTrimmed = 0 + t.doCountStructs(ast) +} + +func (t *Trimmer) doCountStructs(ast *parser.Thrift) { t.structsTrimmed += len(ast.Structs) + len(ast.Includes) + len(ast.Services) + len(ast.Unions) + len(ast.Exceptions) for _, v := range ast.Structs { t.fieldsTrimmed += len(v.Fields) @@ -210,7 +218,7 @@ func (t *Trimmer) countStructs(ast *parser.Thrift) { t.fieldsTrimmed += len(v.Fields) } for _, v := range ast.Includes { - t.countStructs(v.Reference) + t.doCountStructs(v.Reference) } } @@ -221,7 +229,7 @@ func newTrimmer(files []string, outDir string) (*Trimmer, error) { outDir: outDir, } trimmer.asts = make(map[string]*parser.Thrift) - trimmer.marks = make(map[string]map[interface{}]bool) + trimmer.marks = make(map[string]map[string]bool) pattern := `(?m)^[\s]*(\/\/|#)[\s]*@preserve[\s]*$` trimmer.preserveRegex = regexp.MustCompile(pattern) return trimmer, nil diff --git a/tool/trimmer/trim/trimmer_test.go b/tool/trimmer/trim/trimmer_test.go index 43c89943..2ec0dfef 100644 --- a/tool/trimmer/trim/trimmer_test.go +++ b/tool/trimmer/trim/trimmer_test.go @@ -39,8 +39,8 @@ func TestSingleFile(t *testing.T) { check(err) check(semantic.ResolveSymbols(ast)) trimmer.asts[filename] = ast - trimmer.markAST(ast) - trimmer.traversal(ast, ast.Filename) + trimmer.markAST(ast, nil) + trimmer.traversal(ast) test.Assert(t, len(ast.Structs) == 7) test.Assert(t, len(ast.Includes) == 2) @@ -66,8 +66,8 @@ func TestInclude(t *testing.T) { check(err) check(semantic.ResolveSymbols(ast)) trimmer.asts[filename] = ast - trimmer.markAST(ast) - trimmer.traversal(ast, ast.Filename) + trimmer.markAST(ast, nil) + trimmer.traversal(ast) if path := parser.CircleDetect(ast); len(path) > 0 { check(fmt.Errorf("found include circle:\n\t%s", path)) } @@ -105,6 +105,45 @@ func TestTrimMethod(t *testing.T) { test.Assert(t, len(ast.Services[0].Functions) == 1) } +func TestTrimMethodWithExtend(t *testing.T) { + filename := filepath.Join("..", "test_cases", "test_extend", "common1.thrift") + ast, err := parser.ParseFile(filename, nil, true) + check(err) + if path := parser.CircleDetect(ast); len(path) > 0 { + check(fmt.Errorf("found include circle:\n\t%s", path)) + } + checker := semantic.NewChecker(semantic.Options{FixWarnings: true}) + _, err = checker.CheckAll(ast) + check(err) + check(semantic.ResolveSymbols(ast)) + + methods := make([]string, 1) + methods[0] = "Echo" + + _, err = TrimAST(&TrimASTArg{ + Ast: ast, + TrimMethods: methods, + Preserve: nil, + }) + check(err) + // for common1.thrift, Common1Struct1 and ProcessCommon1 are trimmed + test.Assert(t, len(ast.Structs) <= 0) + test.Assert(t, len(ast.Services) == 1) + test.Assert(t, len(ast.Services[0].Functions) <= 0) + + // for common2.thrift, Common2Struct1 and ProcessCommon2 are trimmed + common2AST := ast.Includes[0].Reference + test.Assert(t, len(common2AST.Structs) <= 0) + test.Assert(t, len(common2AST.Services) == 1) + test.Assert(t, len(common2AST.Services[0].Functions) <= 0) + + // for common3.thrift, Common3Struct1 and ProcessCommon3 are trimmed, Echo and Common3Struct2 are preserved + common3AST := common2AST.Includes[0].Reference + test.Assert(t, len(common3AST.Structs) == 1) + test.Assert(t, len(common3AST.Services) == 1) + test.Assert(t, len(common3AST.Services[0].Functions) == 1) +} + func TestPreserve(t *testing.T) { filename := filepath.Join("..", "test_cases", "tests", "dir", "dir2", "test.thrift") ast, err := parser.ParseFile(filename, nil, true) @@ -127,3 +166,170 @@ func TestPreserve(t *testing.T) { check(err) test.Assert(t, len(ast.Structs) == 0) } + +func TestTrimASTWithCompose(t *testing.T) { + testcases := []struct { + desc string + composeArg func() *TrimASTWithComposeArg + expect func(t *testing.T, ast *parser.Thrift, res *TrimResultInfo, err error) + }{ + { + desc: "two unrelated idls refer to a common idl", + composeArg: func() *TrimASTWithComposeArg { + test1 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test1.thrift") + test2 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test2.thrift") + test1AST := parseAndCheckAST(test1, nil, true) + return &TrimASTWithComposeArg{ + Cfg: &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + test1: nil, + test2: nil, + }, + }, + TargetAST: test1AST, + } + }, + expect: func(t *testing.T, ast *parser.Thrift, res *TrimResultInfo, err error) { + // all the elements have been preserved + test.Assert(t, res.StructsTrimmed == 0) + test.Assert(t, res.FieldsTrimmed == 0) + test.Assert(t, res.StructsTotal == 5) + test.Assert(t, res.FieldsTotal == 5) + test.Assert(t, err == nil) + // all the content in common.thrift has been marked to be preserved + commonAst1 := ast.Includes[0].Reference + test.Assert(t, len(commonAst1.Enums) == 2) + test.Assert(t, len(commonAst1.Structs) == 2) + }, + }, + { + desc: "two unrelated idls refer to two common idls", + composeArg: func() *TrimASTWithComposeArg { + test3 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test3.thrift") + test4 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test4.thrift") + test3AST := parseAndCheckAST(test3, nil, true) + return &TrimASTWithComposeArg{ + Cfg: &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + test3: nil, + test4: nil, + }, + }, + TargetAST: test3AST, + } + }, + expect: func(t *testing.T, ast *parser.Thrift, res *TrimResultInfo, err error) { + // all the elements have been preserved + test.Assert(t, res.StructsTrimmed == 0) + test.Assert(t, res.FieldsTrimmed == 0) + test.Assert(t, res.StructsTotal == 8) + test.Assert(t, res.FieldsTotal == 9) + test.Assert(t, err == nil) + commonAst1 := ast.Includes[0].Reference + test.Assert(t, len(commonAst1.Enums) == 2) + test.Assert(t, len(commonAst1.Structs) == 2) + commonAst2 := ast.Includes[1].Reference + test.Assert(t, len(commonAst2.Enums) == 2) + test.Assert(t, len(commonAst2.Structs) == 2) + + }, + }, + { + desc: "use local idl_compose.yaml", + composeArg: func() *TrimASTWithComposeArg { + test1 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test1.thrift") + test1Ast := parseAndCheckAST(test1, nil, true) + return &TrimASTWithComposeArg{ + TargetAST: test1Ast, + ReadCfgFromLocal: true, + } + }, + expect: func(t *testing.T, ast *parser.Thrift, res *TrimResultInfo, err error) { + // all the elements have been preserved + test.Assert(t, res.StructsTrimmed == 0) + test.Assert(t, res.FieldsTrimmed == 0) + test.Assert(t, err == nil) + // all the content in common.thrift has been marked to be preserved + commonAst1 := ast.Includes[0].Reference + test.Assert(t, len(commonAst1.Enums) == 2) + test.Assert(t, len(commonAst1.Structs) == 2) + }, + }, + { + desc: "test5.thrift refer to test6.thrift, they both refer to two common idls", + composeArg: func() *TrimASTWithComposeArg { + test5 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test5.thrift") + test6 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test6.thrift") + test5AST := parseAndCheckAST(test5, nil, true) + return &TrimASTWithComposeArg{ + Cfg: &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + test5: nil, + test6: nil, + }, + }, + TargetAST: test5AST, + } + }, + expect: func(t *testing.T, ast *parser.Thrift, res *TrimResultInfo, err error) { + // all the elements have been preserved + test.Assert(t, res.StructsTrimmed == 0) + test.Assert(t, res.FieldsTrimmed == 0) + test.Assert(t, err == nil) + // common parts + commonAST1 := ast.Includes[0].Reference + test.Assert(t, len(commonAST1.Enums) == 2) + test.Assert(t, len(commonAST1.Structs) == 2) + commonAST2 := ast.Includes[1].Reference + test.Assert(t, len(commonAST2.Enums) == 2) + test.Assert(t, len(commonAST2.Structs) == 2) + // self + test.Assert(t, len(ast.Structs) == 2) + }, + }, + { + desc: "two unrelated idls refer to two common idls with preserved methods and structs", + composeArg: func() *TrimASTWithComposeArg { + test7 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test7.thrift") + test8 := filepath.Join("..", "test_cases", "multiple_idls", "multiple_idls_include_common", "test8.thrift") + test7AST := parseAndCheckAST(test7, nil, true) + return &TrimASTWithComposeArg{ + Cfg: &IDLComposeArguments{ + IDLs: map[string]*IDLArguments{ + test7: { + Trimmer: &YamlArguments{ + Methods: []string{"Process"}, + PreservedStructs: []string{"Test7Struct3"}, + }, + }, + test8: nil, + }, + }, + TargetAST: test7AST, + } + }, + expect: func(t *testing.T, ast *parser.Thrift, res *TrimResultInfo, err error) { + test.Assert(t, res.StructsTrimmed == 0, res.StructsTrimmed) + // Echo Method has been trimmed + test.Assert(t, res.FieldsTrimmed == 1) + test.Assert(t, res.StructsTotal == 10) + test.Assert(t, res.FieldsTotal == 12) + test.Assert(t, err == nil) + commonAst1 := ast.Includes[0].Reference + test.Assert(t, len(commonAst1.Enums) == 2) + test.Assert(t, len(commonAst1.Structs) == 2) + commonAst2 := ast.Includes[1].Reference + test.Assert(t, len(commonAst2.Enums) == 2) + test.Assert(t, len(commonAst2.Structs) == 2) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + arg := tc.composeArg() + res, err := TrimASTWithCompose(arg) + tc.expect(t, arg.TargetAST, res, err) + }) + } +} diff --git a/tool/trimmer/trim/util.go b/tool/trimmer/trim/util.go new file mode 100644 index 00000000..5c99cc91 --- /dev/null +++ b/tool/trimmer/trim/util.go @@ -0,0 +1,38 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trim + +import ( + "fmt" + "github.com/cloudwego/thriftgo/parser" + "github.com/cloudwego/thriftgo/semantic" +) + +func parseAndCheckAST(path string, includeDirs []string, recursive bool) *parser.Thrift { + ast, err := parser.ParseFile(path, includeDirs, recursive) + check(err) + checkAST(ast) + return ast +} + +func checkAST(ast *parser.Thrift) { + if path := parser.CircleDetect(ast); len(path) > 0 { + check(fmt.Errorf("found include circle:\n\t%s", path)) + } + checker := semantic.NewChecker(semantic.Options{FixWarnings: true}) + _, err := checker.CheckAll(ast) + check(err) + check(semantic.ResolveSymbols(ast)) +}