From 256740190e186a759daf4a08be43734757907786 Mon Sep 17 00:00:00 2001 From: Ruojun Meng Date: Tue, 28 Nov 2023 22:48:13 +0800 Subject: [PATCH 1/6] fix: fix the slow xml marshalling when returning a large list of objects --- go.mod | 27 +- go.sum | 44 +- modular/gater/metadata_handler.go | 163 ++-- modular/gater/metadata_handler_test.go | 1002 ++++++++++++++++++++++++ store/bsdb/prefix.go | 20 +- 5 files changed, 1097 insertions(+), 159 deletions(-) create mode 100644 modular/gater/metadata_handler_test.go diff --git a/go.mod b/go.mod index 1763bafcf..afe0363d1 100644 --- a/go.mod +++ b/go.mod @@ -44,14 +44,19 @@ require ( go.uber.org/mock v0.2.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.10.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/time v0.3.0 - google.golang.org/grpc v1.56.1 + google.golang.org/grpc v1.58.3 gorm.io/driver/mysql v1.4.6 gorm.io/gorm v1.24.5 ) +require ( + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect +) + require ( cosmossdk.io/api v0.4.0 // indirect cosmossdk.io/core v0.6.1 // indirect @@ -124,7 +129,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.3.0 // indirect github.com/golang/glog v1.1.0 // indirect - github.com/golang/mock v1.6.0 // indirect + github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect @@ -275,14 +280,14 @@ require ( go.uber.org/fx v1.19.2 // indirect golang.org/x/arch v0.5.0 // indirect golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.11.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.10.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -296,12 +301,12 @@ require ( replace ( cosmossdk.io/api => github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210 - cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230816082903-b48770f5e210 + cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231128142753-a26f49fc53e2 github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.23.0 github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v0.0.0-20231030090949-99ef7dbd1e62 github.com/cometbft/cometbft-db => github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 - github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231101023808-a3c24a58eca0 + github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231128142753-a26f49fc53e2 github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1 github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20231027060606-82070b5da95a github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 diff --git a/go.sum b/go.sum index 1362b6bf2..4bc3dee92 100644 --- a/go.sum +++ b/go.sum @@ -184,12 +184,12 @@ github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 h1:XcWulGacHVRiSCx90Q github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1/go.mod h1:ey1CiK4bYo1RBNJLRiVbYr5CMdSxci9S/AZRINLtppI= github.com/bnb-chain/greenfield-common/go v0.0.0-20230906132736-eb2f0efea228 h1:WywBaew30hZuqDTOQbkRV3uBg6XHjNIE1s3AXVXbG+8= github.com/bnb-chain/greenfield-common/go v0.0.0-20230906132736-eb2f0efea228/go.mod h1:K9jK80fbahciC+FAvrch8Qsbw9ZkvVgjfKsqrzPTAVA= -github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231101023808-a3c24a58eca0 h1:5BQrQRqQ0GTR3UgvTtlTCJAYPkWrjk9Yk9kRnmzzut0= -github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231101023808-a3c24a58eca0/go.mod h1:ZWyfWX032fdHkICmEoJwylfqmL+Atf/QNVS8GzJq1Kc= +github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231128142753-a26f49fc53e2 h1:npknywabPQ5ujSZsRHpDzn6HgtijKvhB1m8QLENiTF8= +github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231128142753-a26f49fc53e2/go.mod h1:Yrvq+J1Lsm7OHFX+M/AZWBTGt1TRHUTC4VYOMlvW3fs= github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210 h1:GHPbV2bC+gmuO6/sG0Tm8oGal3KKSRlyE+zPscDjlA8= github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210/go.mod h1:vhsZxXE9tYJeYB5JR4hPhd6Pc/uPf7j1T8IJ7p9FdeM= -github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230816082903-b48770f5e210 h1:FLVOn4+OVbsKi2+YJX5kmD27/4dRu4FW7xCXFhzDO5s= -github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230816082903-b48770f5e210/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM= +github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231128142753-a26f49fc53e2 h1:YTmIxmvuvMNTVv9HpdVWkgEfe2mbq7HhURG18CnBA4g= +github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231128142753-a26f49fc53e2/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM= github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM= github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI= github.com/bnb-chain/juno/v4 v4.0.0-20231027060606-82070b5da95a h1:fEv/SCzaoumMqfY6OcP4ksXR2L0D8DPbIXPfXdXiODg= @@ -1734,8 +1734,8 @@ golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1851,8 +1851,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2001,16 +2001,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2025,8 +2025,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2203,8 +2203,12 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210426193834-eac7f76ac494/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -2238,8 +2242,8 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20200723182653-9106c3fff523/go.mod h1:5j1uub0jRGhRiSghIlrThmBUgcgLXOVJQ/l1getT4uo= google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= @@ -2257,8 +2261,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= diff --git a/modular/gater/metadata_handler.go b/modular/gater/metadata_handler.go index 97c81e6cf..63f6def18 100644 --- a/modular/gater/metadata_handler.go +++ b/modular/gater/metadata_handler.go @@ -3,16 +3,15 @@ package gater import ( "bytes" "encoding/base64" + "encoding/json" "encoding/xml" - "fmt" + "github.com/cosmos/gogoproto/jsonpb" "net/http" "net/url" - "regexp" "strconv" "strings" "time" - "github.com/cosmos/gogoproto/jsonpb" "github.com/ethereum/go-ethereum/common" "github.com/gorilla/mux" @@ -168,6 +167,7 @@ func (g *GateModular) listObjectsByBucketNameHandler(w http.ResponseWriter, r *h requestDelimiter = queryParams.Get(ListObjectsDelimiterQuery) requestPrefix = queryParams.Get(ListObjectsPrefixQuery) requestIncludeRemoved = queryParams.Get(ListObjectsIncludeRemovedQuery) + format := queryParams.Get("format") if requestDelimiter != "" && requestDelimiter != "/" { log.CtxErrorw(reqCtx.Context(), "failed to check delimiter", "delimiter", requestDelimiter, "error", err) @@ -277,14 +277,21 @@ func (g *GateModular) listObjectsByBucketNameHandler(w http.ResponseWriter, r *h ContinuationToken: continuationToken, } - respBytes, err = xml.Marshal(grpcResponse) + if format == "json" { + respBytes, err = json.Marshal(grpcResponse) + + w.Header().Set(ContentTypeHeader, ContentTypeJSONHeaderValue) + w.Write(respBytes) + return + } + + respBytes, err = xml.Marshal((*GfSpListObjectsByBucketNameResponse)(grpcResponse)) + if err != nil { log.CtxErrorw(reqCtx.Context(), "failed to get user buckets", "error", err) return } - respBytes = processObjectsXmlResponse(respBytes, grpcResponse.Objects) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -335,16 +342,13 @@ func (g *GateModular) getObjectMetaHandler(w http.ResponseWriter, r *http.Reques grpcResponse := &types.GfSpGetObjectMetaResponse{ Object: resp, } + respBytes, err = xml.Marshal((*GfSpGetObjectMetaResponse)(grpcResponse)) - respBytes, err = xml.Marshal(grpcResponse) if err != nil { log.Errorf("failed to get object meta", "error", err) return } - var objects = []*types.Object{grpcResponse.Object} - respBytes = processObjectsXmlResponse(respBytes, objects) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -611,6 +615,9 @@ func (m GfSpListObjectsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.Start } for k, v := range m.Objects { + for i, c := range v.ObjectInfo.Checksums { + v.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + } e.Encode(ObjectEntry{Id: k, Value: v}) } @@ -663,7 +670,7 @@ func (g *GateModular) listObjectsByIDsHandler(w http.ResponseWriter, r *http.Req } if len(objectIDs) == 0 || len(objectIDs) > MaximumIDSize { - log.Errorf("failed to check ids", "error", err) + log.Errorf("len(objectIDs) is invalid : %d", len(objectIDs)) err = ErrInvalidQuery return } @@ -693,8 +700,6 @@ func (g *GateModular) listObjectsByIDsHandler(w http.ResponseWriter, r *http.Req return } - respBytes = processObjectsMapXmlResponse(respBytes, grpcResponse.Objects) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2522,7 +2527,7 @@ func (g *GateModular) listPaymentAccountStreamsHandler(w http.ResponseWriter, r if ok := common.IsHexAddress(paymentAccount); !ok { log.Errorw("failed to check payment account", "payment-account", paymentAccount, "error", err) - err = ErrInvalidHeader + err = ErrInvalidQuery return } @@ -2540,8 +2545,6 @@ func (g *GateModular) listPaymentAccountStreamsHandler(w http.ResponseWriter, r return } - respBytes = processBucketsXmlResponse(respBytes, grpcResponse.Buckets) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2590,8 +2593,6 @@ func (g *GateModular) listUserPaymentAccountsHandler(w http.ResponseWriter, r *h return } - respBytes = processPaymentResponse(respBytes, grpcResponse) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2620,6 +2621,33 @@ func (m GfSpListGroupsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartE return e.EncodeToken(start.End()) } +type GfSpListObjectsByBucketNameResponse types.GfSpListObjectsByBucketNameResponse + +func (m GfSpListObjectsByBucketNameResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type Alias GfSpListObjectsByBucketNameResponse + // Create a new struct with Base64-encoded Checksums field + responseAlias := Alias(m) + for _, o := range responseAlias.Objects { + for i, c := range o.ObjectInfo.Checksums { + o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + } + } + return e.EncodeElement(responseAlias, start) +} + +type GfSpGetObjectMetaResponse types.GfSpGetObjectMetaResponse + +func (m GfSpGetObjectMetaResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type Alias GfSpGetObjectMetaResponse + // Create a new struct with Base64-encoded Checksums field + responseAlias := Alias(m) + o := responseAlias.Object + for i, c := range o.ObjectInfo.Checksums { + o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + } + return e.EncodeElement(responseAlias, start) +} + // listGroupsByIDsHandler list groups by ids func (g *GateModular) listGroupsByIDsHandler(w http.ResponseWriter, r *http.Request) { var ( @@ -2696,8 +2724,6 @@ func (g *GateModular) listGroupsByIDsHandler(w http.ResponseWriter, r *http.Requ return } - respBytes = processGroupsMapXmlResponse(respBytes, grpcResponse.Groups) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2882,86 +2908,6 @@ func (g *GateModular) getBucketSizeHandler(w http.ResponseWriter, r *http.Reques w.Write(respBytes) } -// processObjectsXmlResponse process the unhandled Uint id and checksum of object xml unmarshal -func processObjectsXmlResponse(respBytes []byte, objects []*types.Object) (respBytesProcessed []byte) { - respString := string(respBytes) - // startIdx is to trace the index for next checkSum to be processed - var startIdx = 0 - for _, object := range objects { - if object != nil { - // iterate through each object and assign id value - respString = strings.Replace(respString, "", ""+object.ObjectInfo.Id.String()+"", 1) - // inside each object, there is an array of checksum that need to be unmarshalled correctly - for _, checkSum := range object.ObjectInfo.Checksums { - re := regexp.MustCompile("(.*?)") - // first find the matching string of regex - found := re.FindString(respString[startIdx:]) - respCheckSum := "" + fmt.Sprintf("%x", checkSum) + "" - if found != "" { - // replace the matching string of regex and move the startIdx after that to search next regex - respString = strings.Replace(respString, found, respCheckSum, 1) - startIdx = strings.LastIndex(respString, respCheckSum) + len(respCheckSum) - } - } - } - } - respBytesProcessed = []byte(respString) - return -} - -// processObjectsMapXmlResponse process the unhandled Uint id and checksum of object map xml unmarshal -func processObjectsMapXmlResponse(respBytes []byte, objects map[uint64]*types.Object) (respBytesProcessed []byte) { - respString := string(respBytes) - // startIdx is to trace the index for next checkSum to be processed - var startIdx = 0 - for _, object := range objects { - if object != nil { - // iterate through each object and assign id value - respString = strings.Replace(respString, "", ""+object.ObjectInfo.Id.String()+"", 1) - // inside each object, there is an array of checksum that need to be unmarshalled correctly - for _, checkSum := range object.ObjectInfo.Checksums { - re := regexp.MustCompile("(.*?)") - // first find the matching string of regex - found := re.FindString(respString[startIdx:]) - respCheckSum := "" + fmt.Sprintf("%x", checkSum) + "" - if found != "" { - // replace the matching string of regex and move the startIdx after that to search next regex - respString = strings.Replace(respString, found, respCheckSum, 1) - startIdx = strings.LastIndex(respString, respCheckSum) + len(respCheckSum) - } - } - } - } - respBytesProcessed = []byte(respString) - return -} - -// processGroupsMapXmlResponse process the unhandled Uint id of group map xml unmarshal -func processGroupsMapXmlResponse(respBytes []byte, groups map[uint64]*types.Group) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, group := range groups { - if group != nil { - // iterate through each group and assign id value - respString = strings.Replace(respString, "", ""+group.Group.Id.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - -// processBucketsXmlResponse process the unhandled Uint id of bucket xml unmarshal -func processBucketsXmlResponse(respBytes []byte, buckets []*types.Bucket) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, bucket := range buckets { - if bucket != nil { - // iterate through each bucket and assign id value - respString = strings.Replace(respString, "", ""+bucket.BucketInfo.Id.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - // processBucketsWithPaymentResponse process the unhandled Uint id and several balance of bucket with payment xml unmarshal func processBucketsWithPaymentResponse(respBytes []byte, buckets []*types.GfSpGetBucketMetaResponse) (respBytesProcessed []byte) { respString := string(respBytes) @@ -2980,23 +2926,6 @@ func processBucketsWithPaymentResponse(respBytes []byte, buckets []*types.GfSpGe return } -// processPaymentResponse process the unhandled Uint id and several balance of payment xml unmarshal -func processPaymentResponse(respBytes []byte, payments *types.GfSpListUserPaymentAccountsResponse) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, payment := range payments.PaymentAccounts { - if payment != nil { - // iterate through each payment Uint value - respString = strings.Replace(respString, "", ""+payment.StreamRecord.NetflowRate.String()+"", 1) - respString = strings.Replace(respString, "", ""+payment.StreamRecord.StaticBalance.String()+"", 1) - respString = strings.Replace(respString, "", ""+payment.StreamRecord.BufferBalance.String()+"", 1) - respString = strings.Replace(respString, "", ""+payment.StreamRecord.LockBalance.String()+"", 1) - respString = strings.Replace(respString, "", ""+payment.StreamRecord.FrozenNetflowRate.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - // processBucketsMapXmlResponse process the unhandled Uint id of bucket map xml unmarshal func processBucketsMapXmlResponse(respBytes []byte, buckets map[uint64]*types.Bucket) (respBytesProcessed []byte) { respString := string(respBytes) diff --git a/modular/gater/metadata_handler_test.go b/modular/gater/metadata_handler_test.go new file mode 100644 index 000000000..d77d9d031 --- /dev/null +++ b/modular/gater/metadata_handler_test.go @@ -0,0 +1,1002 @@ +package gater + +import ( + "cosmossdk.io/math" + "encoding/base64" + "encoding/json" + "encoding/xml" + "fmt" + "github.com/bnb-chain/greenfield-storage-provider/modular/metadata/types" + payment_types "github.com/bnb-chain/greenfield/x/payment/types" + storage_types "github.com/bnb-chain/greenfield/x/storage/types" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + "math/big" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/bnb-chain/greenfield-storage-provider/base/gfspclient" +) + +const ( + testAccount = "0xF72aDa8130f934887755492879496b026665FbAB" +) + +func mockListObjectsByBucketNameRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + var routers []*mux.Router + routers = append(routers, router.Host("{bucket:.+}."+g.domain).Subrouter()) + routers = append(routers, router.PathPrefix("/{bucket}").Subrouter()) + for _, r := range routers { + r.NewRoute().Name(listObjectsByBucketRouterName).Methods(http.MethodGet).Path("/").HandlerFunc(g.listObjectsByBucketNameHandler) + + } + return router +} + +func getSampleChecksum() [][]byte { + checksumsInBase64 := [7]string{"tPsLBcgLxRVKTRJCeYw5FVj0jjqPsqFnbDCr77pf7RA=", + "7YqCbwK/qC+zaAoJvd971fuJCE0OVQ9ky8bgomUkmRI=", + "i59qS3vgvN8QIcNKOJggtN4JsZRLYt1ugeGDtP6x7Sk=", + "tBBu4BPpANbc12SO5TVeQ64DtKwl0F2inE29H9jAw54=", + "vOw+loeUIXXPEvfYNFmnElTIxj/b0dEEBBF1YbKOoEI=", + "e0nSN4a5u3EDPaAqemGDZ5gYJ0l6NUjtalmj/BH2uWE=", + "rRm6iKPMc8gZbw1WKKF2kPXveU2VFEh2izs9e8ovfwk="} + checksums := make([][]byte, len(checksumsInBase64)) + for j := 0; j < len(checksums); j++ { + checksums[j], _ = base64.StdEncoding.DecodeString(checksumsInBase64[j]) + } + return checksums + +} +func getTestGroupsInIdMap(length int) map[uint64]*types.Group { + groupsMap := make(map[uint64]*types.Group) + groups := getSampleTestGroupsResponse(length) + for _, g := range groups { + groupsMap[g.Group.Id.BigInt().Uint64()] = g + } + return groupsMap +} + +func getTestBuckets(length int) []*types.Bucket { + buckets := make([]*types.Bucket, length) + + for i := 0; i < length; i++ { + bucket := &types.Bucket{ + BucketInfo: &storage_types.BucketInfo{ + Owner: testAccount, + BucketName: mockBucketName + strconv.Itoa(i), + Visibility: storage_types.VISIBILITY_TYPE_PUBLIC_READ, + Id: math.NewUintFromBigInt(big.NewInt(int64(i))), + SourceType: storage_types.SOURCE_TYPE_ORIGIN, + CreateAt: 1699781080, + PaymentAddress: testAccount, + GlobalVirtualGroupFamilyId: 3, + ChargedReadQuota: 0, + BucketStatus: storage_types.BUCKET_STATUS_CREATED, + }, + Removed: false, + DeleteAt: 0, + DeleteReason: "", + Operator: testAccount, + CreateTxHash: "0x21c349a869bde1f44378936e2a9a15ed3fb2d54a43eaea8787960bba1134cdc2", + UpdateTxHash: "0x0cbff0ff3831d61345dbfda5b984e254c4bf87ecf80b45ccbb0635c0547a3b1a", + UpdateAt: 1279811, + UpdateTime: 1699781103, + } + buckets[i] = bucket + } + return buckets +} + +func getTestPaymentAccountMeta() []*types.PaymentAccountMeta { + paymentAccounts := make([]*types.PaymentAccountMeta, 1) + + paymentAccount := &types.PaymentAccountMeta{ + StreamRecord: &payment_types.StreamRecord{ + Account: testAccount, + CrudTimestamp: 1699780994, + NetflowRate: math.NewIntFromBigInt(big.NewInt(int64(0))), + StaticBalance: math.NewIntFromBigInt(big.NewInt(int64(240000000000000001))), + BufferBalance: math.NewIntFromBigInt(big.NewInt(int64(0))), + LockBalance: math.NewIntFromBigInt(big.NewInt(int64(0))), + Status: payment_types.STREAM_ACCOUNT_STATUS_ACTIVE, + SettleTimestamp: 0, + OutFlowCount: 0, + FrozenNetflowRate: math.NewIntFromBigInt(big.NewInt(int64(0))), + }, + PaymentAccount: &types.PaymentAccount{ + Address: testAccount, + Owner: testAccount, + Refundable: true, + UpdateAt: 1279659, + UpdateTime: 1699780707, + }, + } + paymentAccounts[0] = paymentAccount + return paymentAccounts +} + +func getTestObjectsInIdMap(length int) map[uint64]*types.Object { + objectsMap := make(map[uint64]*types.Object) + objects := getTestObjectsResponse(length) + for _, o := range objects { + objectsMap[o.ObjectInfo.Id.BigInt().Uint64()] = o + } + return objectsMap +} + +func getOneTestObjectResponse() *types.Object { + owner := testAccount + object := &types.Object{ + ObjectInfo: &storage_types.ObjectInfo{ + Owner: owner, + Creator: owner, + BucketName: mockBucketName, + ObjectName: mockObjectName, + Id: math.NewUintFromString("24662"), + LocalVirtualGroupId: 1, + PayloadSize: 4802764, + Visibility: storage_types.VISIBILITY_TYPE_INHERIT, + ContentType: "application/octet-stream", + CreateAt: 1699781700, + ObjectStatus: storage_types.OBJECT_STATUS_SEALED, + Checksums: getSampleChecksum(), + }, + + LockedBalance: "0x0000000000000000000000000000000000000000000000000000000000000000", + UpdateAt: 1280048, + Operator: "0x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa0", + CreateTxHash: "0x491227c644bc89f5a058d92167c00d452c63a1dd8d5776c81617a41ec76fcc8c", + UpdateTxHash: "0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e", + SealTxHash: "0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e", + } + return object +} + +func getTestObjectsResponse(objectLength int) []*types.Object { + length := objectLength + objects := make([]*types.Object, length) + owner := testAccount + + for i := 0; i < length; i++ { + object := &types.Object{ + ObjectInfo: &storage_types.ObjectInfo{ + Owner: owner, + Creator: owner, + BucketName: mockBucketName, + ObjectName: mockObjectName + strconv.Itoa(i), + Id: math.NewUintFromBigInt(big.NewInt(int64(i))), + LocalVirtualGroupId: 1, + PayloadSize: 4802764, + Visibility: storage_types.VISIBILITY_TYPE_INHERIT, + ContentType: "application/octet-stream", + CreateAt: 1699781700, + ObjectStatus: storage_types.OBJECT_STATUS_SEALED, + Checksums: getSampleChecksum(), + }, + + LockedBalance: "0x0000000000000000000000000000000000000000000000000000000000000000", + UpdateAt: 1280048, + Operator: "0x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa0", + CreateTxHash: "0x491227c644bc89f5a058d92167c00d452c63a1dd8d5776c81617a41ec76fcc8c", + UpdateTxHash: "0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e", + SealTxHash: "0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e", + } + objects[i] = object + } + return objects +} + +func getSampleTestGroupsResponse(groupLength int) []*types.Group { + length := groupLength + groups := make([]*types.Group, length) + owner := testAccount + + for i := 0; i < length; i++ { + group := &types.Group{ + Group: &storage_types.GroupInfo{ + Owner: owner, + GroupName: "TestGroupName " + strconv.Itoa(i), + SourceType: storage_types.SOURCE_TYPE_ORIGIN, + Id: math.NewUintFromBigInt(big.NewInt(int64(i))), + Extra: "", + }, + NumberOfMembers: 1, + Removed: false, + UpdateAt: 1280048, + Operator: "0x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa0", + } + groups[i] = group + } + return groups +} + +func TestGateModular_ListObjectsByBucketNameHandler(t *testing.T) { + mockData := getTestObjectsResponse(1000) + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListObjectsByBucketName(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(nil, uint64(0), uint64(0), false, "", "", "", "", nil, "", mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/", scheme, mockBucketName, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "wrong requestDelimiter", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=wrong_char&&continuation-token=NjM5LnBuZw==", scheme, mockBucketName, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong BucketName", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&continuation-token=NjM5LnBuZw==", scheme, "aa", testDomain) // aa is an invalid bucket name + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong requestMaxKeys", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=invalidMaxKey&delimiter=%%2F&&continuation-token=NjM5LnBuZw==", scheme, mockBucketName, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong requestStartAfter", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&continuation-token=NjM5LnBuZw==&&start-after=%%2F%%2F", scheme, mockBucketName, testDomain) // %%2F%%2F means "//", which is an invalid start-after + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong requestContinuationToken", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&continuation-token=NjM5LnBuZw==d", scheme, mockBucketName, testDomain) // NjM5LnBuZw==d is an invalid requestContinuationToken + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong requestContinuationToken2", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + invalidContinuationToken := base64.StdEncoding.EncodeToString([]byte("//")) + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&continuation-token=%s", scheme, mockBucketName, testDomain, invalidContinuationToken) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong requestContinuationToken3", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + invalidContinuationToken := base64.StdEncoding.EncodeToString([]byte("not_start_with_prefix")) + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&continuation-token=%s&&prefix=%s", scheme, mockBucketName, testDomain, invalidContinuationToken, "a_sample_prefix") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong requestIncludeRemoved", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&include-removed=invalid", scheme, mockBucketName, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "wrong prefix", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + invalidPrefix := "%2F%2F" // this is an invalid prefix + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&&prefix=%s", scheme, mockBucketName, testDomain, invalidPrefix) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "json response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListObjectsByBucketName(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(mockData, uint64(0), uint64(0), false, "", "", "", "", nil, "", nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&format=json", scheme, mockBucketName, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpListObjectsByBucketNameResponse + err := json.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, len(mockData), len(res.Objects)) + assert.Equal(t, mockData[0].ObjectInfo.Id, res.Objects[0].ObjectInfo.Id) + return true + }, + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListObjectsByBucketName(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(mockData, uint64(0), uint64(0), false, "", "", "", "", nil, "", nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F", scheme, mockBucketName, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpListObjectsByBucketNameResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, len(mockData), len(res.Objects)) + assert.Equal(t, mockData[0].ObjectInfo.Id, res.Objects[0].ObjectInfo.Id) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockListObjectsByBucketNameRoute(t, tt.fn()) + w := httptest.NewRecorder() + begin := time.Now() + router.ServeHTTP(w, tt.request()) + end := time.Now() + assert.Less(t, end.UnixMilli()-begin.UnixMilli(), int64(1000)) // we expected this API can return response in 1 sec after it gets data from DB. + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockGetObjectMetaRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + var routers []*mux.Router + routers = append(routers, router.Host("{bucket:.+}."+g.domain).Subrouter()) + routers = append(routers, router.PathPrefix("/{bucket}").Subrouter()) + for _, r := range routers { + r.NewRoute().Name(getObjectMetaRouterName).Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(g.getObjectMetaHandler). + Queries(GetObjectMetaQuery, "") + } + return router +} + +func TestGateModular_GetObjectMetaHandler(t *testing.T) { + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetObjectMeta(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/%s?%s", scheme, mockBucketName, testDomain, mockObjectName, GetObjectMetaQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid bucket name", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetObjectMeta(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any()).Return(nil, mockErr).Times(0) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/%s?%s", scheme, invalidBucketName, testDomain, mockObjectName, GetObjectMetaQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid object name", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetObjectMeta(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any()).Return(nil, mockErr).Times(0) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/%s?%s", scheme, mockBucketName, testDomain, invalidObjectName, GetObjectMetaQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetObjectMeta(gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any()).Return(getOneTestObjectResponse(), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s.%s/%s?%s", scheme, mockBucketName, testDomain, mockObjectName, GetObjectMetaQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "0xF72aDa8130f934887755492879496b026665FbAB0xF72aDa8130f934887755492879496b026665FbABmock-bucket-namemock-object-name24662148027643application/octet-stream1699781700100tPsLBcgLxRVKTRJCeYw5FVj0jjqPsqFnbDCr77pf7RA=7YqCbwK/qC+zaAoJvd971fuJCE0OVQ9ky8bgomUkmRI=i59qS3vgvN8QIcNKOJggtN4JsZRLYt1ugeGDtP6x7Sk=tBBu4BPpANbc12SO5TVeQ64DtKwl0F2inE29H9jAw54=vOw+loeUIXXPEvfYNFmnElTIxj/b0dEEBBF1YbKOoEI=e0nSN4a5u3EDPaAqemGDZ5gYJ0l6NUjtalmj/BH2uWE=rRm6iKPMc8gZbw1WKKF2kPXveU2VFEh2izs9e8ovfwk=0x0000000000000000000000000000000000000000000000000000000000000000false128004800x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa00x491227c644bc89f5a058d92167c00d452c63a1dd8d5776c81617a41ec76fcc8c0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e", + body) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetObjectMetaRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockListObjectsByIDsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(listObjectsByIDsRouterName).Methods(http.MethodGet).Queries(ListObjectsByIDsQuery, "").HandlerFunc(g.listObjectsByIDsHandler) + return router +} + +func TestGateModular_ListObjectsByIDsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListObjectsByIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListObjectsByIDsQuery, "1,2,3,4") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid id", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListObjectsByIDsQuery, "a,2,3,4") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid id number", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListObjectsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "repeated id number", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "1,1" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListObjectsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListObjectsByIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return(getTestObjectsInIdMap(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "1" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListObjectsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "00xF72aDa8130f934887755492879496b026665FbAB0xF72aDa8130f934887755492879496b026665FbABmock-bucket-namemock-object-name00148027643application/octet-stream1699781700100tPsLBcgLxRVKTRJCeYw5FVj0jjqPsqFnbDCr77pf7RA=7YqCbwK/qC+zaAoJvd971fuJCE0OVQ9ky8bgomUkmRI=i59qS3vgvN8QIcNKOJggtN4JsZRLYt1ugeGDtP6x7Sk=tBBu4BPpANbc12SO5TVeQ64DtKwl0F2inE29H9jAw54=vOw+loeUIXXPEvfYNFmnElTIxj/b0dEEBBF1YbKOoEI=e0nSN4a5u3EDPaAqemGDZ5gYJ0l6NUjtalmj/BH2uWE=rRm6iKPMc8gZbw1WKKF2kPXveU2VFEh2izs9e8ovfwk=0x0000000000000000000000000000000000000000000000000000000000000000false128004800x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa00x491227c644bc89f5a058d92167c00d452c63a1dd8d5776c81617a41ec76fcc8c0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e0x238737f109a40c675e1bef5ebfb2adef2cac0a723ee20fbd752e78efbf3d579e", + body) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockListObjectsByIDsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockListGroupsByIDsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(listGroupsByIDsRouterName).Methods(http.MethodGet).Queries(ListGroupsByIDsQuery, "").HandlerFunc(g.listGroupsByIDsHandler) + return router +} + +func TestGateModular_ListGroupsByIDsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListGroupsByIDs(gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListGroupsByIDsQuery, "1,2,3,4") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid id", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListGroupsByIDsQuery, "a,2,3,4") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid id number", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListGroupsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "repeated id number", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "1,1" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListGroupsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListGroupsByIDs(gomock.Any(), gomock.Any()).Return(getTestGroupsInIdMap(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "1" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListGroupsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "00xF72aDa8130f934887755492879496b026665FbABTestGroupName 0000x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa000128004801false", + body) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockListGroupsByIDsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockListPaymentAccountStreamsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(listPaymentAccountStreamsRouterName).Methods(http.MethodGet).Queries(ListPaymentAccountStreamsQuery, "").HandlerFunc(g.listPaymentAccountStreamsHandler) + return router +} + +func TestGateModular_ListPaymentAccountStreamsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListPaymentAccountStreams(gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&%s=%s", scheme, testDomain, ListPaymentAccountStreamsQuery, PaymentAccountQuery, testAccount) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid payment account", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&%s=%s", scheme, testDomain, ListPaymentAccountStreamsQuery, PaymentAccountQuery, "invalid_payment_account") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListPaymentAccountStreams(gomock.Any(), gomock.Any()).Return(getTestBuckets(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&%s=%s", scheme, testDomain, ListPaymentAccountStreamsQuery, PaymentAccountQuery, testAccount) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "0xF72aDa8130f934887755492879496b026665FbABmock-bucket-name010016997810800xF72aDa8130f934887755492879496b026665FbAB300false00xF72aDa8130f934887755492879496b026665FbAB0x21c349a869bde1f44378936e2a9a15ed3fb2d54a43eaea8787960bba1134cdc20x0cbff0ff3831d61345dbfda5b984e254c4bf87ecf80b45ccbb0635c0547a3b1a12798111699781103", + body) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockListPaymentAccountStreamsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockListUserPaymentAccountsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(listUserPaymentAccountsRouterName).Methods(http.MethodGet).Queries(ListUserPaymentAccountsQuery, "").HandlerFunc(g.listUserPaymentAccountsHandler) + return router +} + +func TestGateModular_ListUserPaymentAccountsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListUserPaymentAccounts(gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s", scheme, testDomain, ListUserPaymentAccountsQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid payment account", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s", scheme, testDomain, ListUserPaymentAccountsQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, "invalid_payment_account") + + return req + }, + wantedResult: "invalid request header", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListUserPaymentAccounts(gomock.Any(), gomock.Any()).Return(getTestPaymentAccountMeta(), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s", scheme, testDomain, ListUserPaymentAccountsQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "
0xF72aDa8130f934887755492879496b026665FbAB
0xF72aDa8130f934887755492879496b026665FbABtrue127965916997807070xF72aDa8130f934887755492879496b026665FbAB16997809940240000000000000001000000", + body) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockListUserPaymentAccountsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} diff --git a/store/bsdb/prefix.go b/store/bsdb/prefix.go index 5b0b841d5..e877b57b4 100644 --- a/store/bsdb/prefix.go +++ b/store/bsdb/prefix.go @@ -56,7 +56,7 @@ func (b *BsDBImpl) ListObjects(bucketName, continuationToken, prefix string, max if err != nil { return nil, err } - res, err = b.filterObjects(nodes) + res, err = b.filterObjects(nodes, bucketName) return res, err } @@ -82,7 +82,7 @@ func processPath(pathName string) (string, string) { // filterObjects filters a slice of SlashPrefixTreeNode for nodes which IsObject attribute is true, // maps these objects by their ID and transforms them into a ListObjectsResult format. // Returns a slice of ListObjectsResult containing filtered object data or an error if something goes wrong. -func (b *BsDBImpl) filterObjects(nodes []*SlashPrefixTreeNode) ([]*ListObjectsResult, error) { +func (b *BsDBImpl) filterObjects(nodes []*SlashPrefixTreeNode, bucketName string) ([]*ListObjectsResult, error) { var ( objectIDs []common.Hash totalObjects []*Object @@ -108,16 +108,14 @@ func (b *BsDBImpl) filterObjects(nodes []*SlashPrefixTreeNode) ([]*ListObjectsRe } } - for i := 0; i < ObjectsNumberOfShards; i++ { - err = b.db.Table(GetObjectsTableNameByShardNumber(i)). - Where("object_id in (?)", objectIDs). - Find(&objects).Error - //stop after finding one set? - if err != nil { - return nil, err - } - totalObjects = append(totalObjects, objects...) + err = b.db.Table(GetObjectsTableName(bucketName)). + Where("object_id in (?)", objectIDs). + Find(&objects).Error + //stop after finding one set? + if err != nil { + return nil, err } + totalObjects = append(totalObjects, objects...) objectsMap = make(map[common.Hash]*Object) for _, object := range totalObjects { From e2eaf1d7279cc837e34c41187b5136dc9d0c84bb Mon Sep 17 00:00:00 2001 From: Ruojun Meng Date: Wed, 29 Nov 2023 15:20:23 +0800 Subject: [PATCH 2/6] fix: remove unnecessary xml modification and add UTs --- go.mod | 6 +- go.sum | 12 +- modular/gater/metadata_handler.go | 87 +- modular/gater/metadata_handler_test.go | 1108 +++++++++++++++++++++++- 4 files changed, 1104 insertions(+), 109 deletions(-) diff --git a/go.mod b/go.mod index afe0363d1..b176cd9f1 100644 --- a/go.mod +++ b/go.mod @@ -300,13 +300,13 @@ require ( ) replace ( - cosmossdk.io/api => github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210 - cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231128142753-a26f49fc53e2 + cosmossdk.io/api => github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231129013257-1e407f209b02 + cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231129013257-1e407f209b02 github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.23.0 github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v0.0.0-20231030090949-99ef7dbd1e62 github.com/cometbft/cometbft-db => github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 - github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231128142753-a26f49fc53e2 + github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231129013257-1e407f209b02 github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1 github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20231027060606-82070b5da95a github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 diff --git a/go.sum b/go.sum index 4bc3dee92..f304f7fe8 100644 --- a/go.sum +++ b/go.sum @@ -184,12 +184,12 @@ github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 h1:XcWulGacHVRiSCx90Q github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1/go.mod h1:ey1CiK4bYo1RBNJLRiVbYr5CMdSxci9S/AZRINLtppI= github.com/bnb-chain/greenfield-common/go v0.0.0-20230906132736-eb2f0efea228 h1:WywBaew30hZuqDTOQbkRV3uBg6XHjNIE1s3AXVXbG+8= github.com/bnb-chain/greenfield-common/go v0.0.0-20230906132736-eb2f0efea228/go.mod h1:K9jK80fbahciC+FAvrch8Qsbw9ZkvVgjfKsqrzPTAVA= -github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231128142753-a26f49fc53e2 h1:npknywabPQ5ujSZsRHpDzn6HgtijKvhB1m8QLENiTF8= -github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231128142753-a26f49fc53e2/go.mod h1:Yrvq+J1Lsm7OHFX+M/AZWBTGt1TRHUTC4VYOMlvW3fs= -github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210 h1:GHPbV2bC+gmuO6/sG0Tm8oGal3KKSRlyE+zPscDjlA8= -github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210/go.mod h1:vhsZxXE9tYJeYB5JR4hPhd6Pc/uPf7j1T8IJ7p9FdeM= -github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231128142753-a26f49fc53e2 h1:YTmIxmvuvMNTVv9HpdVWkgEfe2mbq7HhURG18CnBA4g= -github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231128142753-a26f49fc53e2/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM= +github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231129013257-1e407f209b02 h1:x9F+rt7H4VOS4JfoqMFFvq3RGW27oSxIFPTI/cMK4tQ= +github.com/bnb-chain/greenfield-cosmos-sdk v1.0.2-0.20231129013257-1e407f209b02/go.mod h1:Yrvq+J1Lsm7OHFX+M/AZWBTGt1TRHUTC4VYOMlvW3fs= +github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231129013257-1e407f209b02 h1:cwuShQ+MlvwkfbOz79BRF4aYjgKAuSyugoCtXE8tWgM= +github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20231129013257-1e407f209b02/go.mod h1:vhsZxXE9tYJeYB5JR4hPhd6Pc/uPf7j1T8IJ7p9FdeM= +github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231129013257-1e407f209b02 h1:RMrZep7e2dcMk4E5YysMdCn6PekI3vA2WHfnMQ/kg18= +github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231129013257-1e407f209b02/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM= github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM= github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI= github.com/bnb-chain/juno/v4 v4.0.0-20231027060606-82070b5da95a h1:fEv/SCzaoumMqfY6OcP4ksXR2L0D8DPbIXPfXdXiODg= diff --git a/modular/gater/metadata_handler.go b/modular/gater/metadata_handler.go index 63f6def18..f90c0783c 100644 --- a/modular/gater/metadata_handler.go +++ b/modular/gater/metadata_handler.go @@ -117,8 +117,6 @@ func (g *GateModular) getUserBucketsHandler(w http.ResponseWriter, r *http.Reque return } - respBytes = processVGFInfoBucketXmlResponse(respBytes, grpcResponse.Buckets) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -399,9 +397,6 @@ func (g *GateModular) getBucketMetaHandler(w http.ResponseWriter, r *http.Reques return } - var bucketsWithPayment = []*types.GfSpGetBucketMetaResponse{grpcResponse} - respBytes = processBucketsWithPaymentResponse(respBytes, bucketsWithPayment) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -591,8 +586,6 @@ func (g *GateModular) getGroupListHandler(w http.ResponseWriter, r *http.Request return } - respBytes = processGroupsXmlResponse(respBytes, grpcResponse.Groups) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -804,8 +797,6 @@ func (g *GateModular) listBucketsByIDsHandler(w http.ResponseWriter, r *http.Req return } - respBytes = processBucketsMapXmlResponse(respBytes, grpcResponse.Buckets) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2244,8 +2235,6 @@ func (g *GateModular) getUserGroupsHandler(w http.ResponseWriter, r *http.Reques return } - respBytes = processGroupMembersXmlResponse(respBytes, grpcResponse.Groups) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2288,7 +2277,7 @@ func (g *GateModular) getGroupMembersHandler(w http.ResponseWriter, r *http.Requ if requestStartAfter != "" { if ok := common.IsHexAddress(requestStartAfter); !ok { log.Errorw("failed to check start after", "start-after", requestStartAfter, "error", err) - err = ErrInvalidHeader + err = ErrInvalidQuery return } } @@ -2321,8 +2310,6 @@ func (g *GateModular) getGroupMembersHandler(w http.ResponseWriter, r *http.Requ return } - respBytes = processGroupMembersXmlResponse(respBytes, grpcResponse.Groups) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2396,8 +2383,6 @@ func (g *GateModular) getUserOwnedGroupsHandler(w http.ResponseWriter, r *http.R return } - respBytes = processGroupMembersXmlResponse(respBytes, grpcResponse.Groups) - w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } @@ -2907,73 +2892,3 @@ func (g *GateModular) getBucketSizeHandler(w http.ResponseWriter, r *http.Reques w.Header().Set(ContentTypeHeader, ContentTypeXMLHeaderValue) w.Write(respBytes) } - -// processBucketsWithPaymentResponse process the unhandled Uint id and several balance of bucket with payment xml unmarshal -func processBucketsWithPaymentResponse(respBytes []byte, buckets []*types.GfSpGetBucketMetaResponse) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, bucket := range buckets { - if bucket != nil { - // iterate through each bucket and assign id and payment Uint value - respString = strings.Replace(respString, "", ""+bucket.Bucket.BucketInfo.Id.String()+"", 1) - respString = strings.Replace(respString, "", ""+bucket.StreamRecord.NetflowRate.String()+"", 1) - respString = strings.Replace(respString, "", ""+bucket.StreamRecord.StaticBalance.String()+"", 1) - respString = strings.Replace(respString, "", ""+bucket.StreamRecord.BufferBalance.String()+"", 1) - respString = strings.Replace(respString, "", ""+bucket.StreamRecord.LockBalance.String()+"", 1) - respString = strings.Replace(respString, "", ""+bucket.StreamRecord.FrozenNetflowRate.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - -// processBucketsMapXmlResponse process the unhandled Uint id of bucket map xml unmarshal -func processBucketsMapXmlResponse(respBytes []byte, buckets map[uint64]*types.Bucket) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, bucket := range buckets { - if bucket != nil { - // iterate through each bucket and assign id value - respString = strings.Replace(respString, "", ""+bucket.BucketInfo.Id.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - -// processGroupsXmlResponse process the unhandled Uint id of group xml unmarshal -func processGroupsXmlResponse(respBytes []byte, groups []*types.Group) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, group := range groups { - if group != nil { - // iterate through each group and assign id value - respString = strings.Replace(respString, "", ""+group.Group.Id.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - -// processGroupMembersXmlResponse process the unhandled Uint id of group member xml unmarshal -func processGroupMembersXmlResponse(respBytes []byte, groupMembers []*types.GroupMember) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, group := range groupMembers { - if group != nil { - // iterate through each group and assign id value - respString = strings.Replace(respString, "", ""+group.Group.Id.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} - -// processVGFInfoBucketXmlResponse process the unhandled Uint id of bucket with vgf xml unmarshal -func processVGFInfoBucketXmlResponse(respBytes []byte, buckets []*types.VGFInfoBucket) (respBytesProcessed []byte) { - respString := string(respBytes) - for _, bucket := range buckets { - if bucket != nil { - // iterate through each bucket and assign id value - respString = strings.Replace(respString, "", ""+bucket.BucketInfo.Id.String()+"", 1) - } - } - respBytesProcessed = []byte(respString) - return -} diff --git a/modular/gater/metadata_handler_test.go b/modular/gater/metadata_handler_test.go index d77d9d031..648836131 100644 --- a/modular/gater/metadata_handler_test.go +++ b/modular/gater/metadata_handler_test.go @@ -9,9 +9,11 @@ import ( "github.com/bnb-chain/greenfield-storage-provider/modular/metadata/types" payment_types "github.com/bnb-chain/greenfield/x/payment/types" storage_types "github.com/bnb-chain/greenfield/x/storage/types" + virtual_types "github.com/bnb-chain/greenfield/x/virtualgroup/types" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + "log" "math/big" "net/http" "net/http/httptest" @@ -57,13 +59,22 @@ func getSampleChecksum() [][]byte { } func getTestGroupsInIdMap(length int) map[uint64]*types.Group { groupsMap := make(map[uint64]*types.Group) - groups := getSampleTestGroupsResponse(length) + groups := getTestGroupsResponse(length) for _, g := range groups { groupsMap[g.Group.Id.BigInt().Uint64()] = g } return groupsMap } +func getTestBucketsInIdMap(length int) map[uint64]*types.Bucket { + bucketsMap := make(map[uint64]*types.Bucket) + buckets := getTestBuckets(length) + for _, b := range buckets { + bucketsMap[b.BucketInfo.Id.BigInt().Uint64()] = b + } + return bucketsMap +} + func getTestBuckets(length int) []*types.Bucket { buckets := make([]*types.Bucket, length) @@ -95,22 +106,62 @@ func getTestBuckets(length int) []*types.Bucket { return buckets } +func getTestVGFInfoBuckets(length int) []*types.VGFInfoBucket { + buckets := make([]*types.VGFInfoBucket, length) + + for i := 0; i < length; i++ { + bucket := &types.VGFInfoBucket{ + BucketInfo: &storage_types.BucketInfo{ + Owner: testAccount, + BucketName: mockBucketName + strconv.Itoa(i), + Visibility: storage_types.VISIBILITY_TYPE_PUBLIC_READ, + Id: math.NewUintFromBigInt(big.NewInt(int64(i))), + SourceType: storage_types.SOURCE_TYPE_ORIGIN, + CreateAt: 1699781080, + PaymentAddress: testAccount, + GlobalVirtualGroupFamilyId: 3, + ChargedReadQuota: 0, + BucketStatus: storage_types.BUCKET_STATUS_CREATED, + }, + Removed: false, + DeleteAt: 0, + DeleteReason: "", + Operator: testAccount, + CreateTxHash: "0x21c349a869bde1f44378936e2a9a15ed3fb2d54a43eaea8787960bba1134cdc2", + UpdateTxHash: "0x0cbff0ff3831d61345dbfda5b984e254c4bf87ecf80b45ccbb0635c0547a3b1a", + UpdateAt: 1279811, + UpdateTime: 1699781103, + Vgf: &virtual_types.GlobalVirtualGroupFamily{ + Id: uint32(i), + PrimarySpId: uint32(3), + VirtualPaymentAddress: "0x26281179b8885F21f95b0a246c8AD70957A95A23", + }, + } + buckets[i] = bucket + } + return buckets +} + +func getTestStreamRecord() *payment_types.StreamRecord { + return &payment_types.StreamRecord{ + Account: testAccount, + CrudTimestamp: 1699780994, + NetflowRate: math.NewIntFromBigInt(big.NewInt(int64(0))), + StaticBalance: math.NewIntFromBigInt(big.NewInt(int64(240000000000000001))), + BufferBalance: math.NewIntFromBigInt(big.NewInt(int64(0))), + LockBalance: math.NewIntFromBigInt(big.NewInt(int64(0))), + Status: payment_types.STREAM_ACCOUNT_STATUS_ACTIVE, + SettleTimestamp: 0, + OutFlowCount: 0, + FrozenNetflowRate: math.NewIntFromBigInt(big.NewInt(int64(0))), + } +} + func getTestPaymentAccountMeta() []*types.PaymentAccountMeta { paymentAccounts := make([]*types.PaymentAccountMeta, 1) paymentAccount := &types.PaymentAccountMeta{ - StreamRecord: &payment_types.StreamRecord{ - Account: testAccount, - CrudTimestamp: 1699780994, - NetflowRate: math.NewIntFromBigInt(big.NewInt(int64(0))), - StaticBalance: math.NewIntFromBigInt(big.NewInt(int64(240000000000000001))), - BufferBalance: math.NewIntFromBigInt(big.NewInt(int64(0))), - LockBalance: math.NewIntFromBigInt(big.NewInt(int64(0))), - Status: payment_types.STREAM_ACCOUNT_STATUS_ACTIVE, - SettleTimestamp: 0, - OutFlowCount: 0, - FrozenNetflowRate: math.NewIntFromBigInt(big.NewInt(int64(0))), - }, + StreamRecord: getTestStreamRecord(), PaymentAccount: &types.PaymentAccount{ Address: testAccount, Owner: testAccount, @@ -194,7 +245,7 @@ func getTestObjectsResponse(objectLength int) []*types.Object { return objects } -func getSampleTestGroupsResponse(groupLength int) []*types.Group { +func getTestGroupsResponse(groupLength int) []*types.Group { length := groupLength groups := make([]*types.Group, length) owner := testAccount @@ -218,6 +269,32 @@ func getSampleTestGroupsResponse(groupLength int) []*types.Group { return groups } +func getTestGroupMembersResponse(groupLength int) []*types.GroupMember { + length := groupLength + groupMembers := make([]*types.GroupMember, length) + for i := 0; i < length; i++ { + groupMember := &types.GroupMember{ + Group: &storage_types.GroupInfo{ + Owner: testAccount, + GroupName: "TestGroupName " + strconv.Itoa(i), + SourceType: storage_types.SOURCE_TYPE_ORIGIN, + Id: math.NewUintFromBigInt(big.NewInt(int64(i))), + Extra: "", + }, + AccountId: testAccount, + Operator: testAccount, + CreateAt: 1376086, + CreateTime: 1700032197, + UpdateAt: 1376086, + UpdateTime: 1700032197, + Removed: false, + ExpirationTime: 253402300799, + } + groupMembers[i] = groupMember + } + return groupMembers +} + func TestGateModular_ListObjectsByBucketNameHandler(t *testing.T) { mockData := getTestObjectsResponse(1000) cases := []struct { @@ -823,6 +900,227 @@ func TestGateModular_ListGroupsByIDsHandler(t *testing.T) { } } +func mockGetGroupListHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(getGroupListRouterName).Methods(http.MethodGet).Queries(GetGroupListGroupQuery, "").HandlerFunc(g.getGroupListHandler) + return router +} + +func TestGateModular_GetGroupListHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, int64(0), mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=10&offset=0&name=e&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "blank name", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=10&offset=0&name=&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "blank prefix", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=10&offset=0&name=n&prefix=", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "bad offset", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=&offset=a&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "bad limit", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=a&offset=&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "limit is negative", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=-1&offset=&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "bad sourceType", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_UNKNOWN&limit=1&offset=0&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(1), int64(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=1&offset=0&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "0xF72aDa8130f934887755492879496b026665FbABTestGroupName 0000x03AbbEe8E426C9887A8ae3C34602AbCA42aeDFa000128004801false1", + body) + return true + }, + }, + + { + name: "xml response for large limit", // return 1000 records when limit is more than 1000 + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Eq(int64(1000)), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(1000), int64(1), nil).AnyTimes() + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Eq(int64(1001)), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(1001), int64(1), nil).AnyTimes() + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Eq(int64(50)), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(50), int64(1), nil).AnyTimes() + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&limit=1001&offset=0&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpGetGroupListResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1000, len(res.Groups)) + return true + }, + }, + { + name: "xml response for default limit", // return 50 records when limit is not set + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Eq(int64(1000)), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(1000), int64(1), nil).AnyTimes() + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Eq(int64(1001)), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(1001), int64(1), nil).AnyTimes() + clientMock.EXPECT().GetGroupList(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Eq(int64(50)), gomock.Any(), gomock.Any()).Return(getTestGroupsResponse(50), int64(1), nil).AnyTimes() + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-query&source-type=SOURCE_TYPE_ORIGIN&offset=0&name=n&prefix=t", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpGetGroupListResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 50, len(res.Groups)) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetGroupListHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + func mockListPaymentAccountStreamsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { t.Helper() router := mux.NewRouter().SkipClean(true) @@ -1000,3 +1298,785 @@ func TestGateModular_ListUserPaymentAccountsHandler(t *testing.T) { }) } } + +func mockListBucketsByIDsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(listBucketsByIDsRouterName).Methods(http.MethodGet).Queries(ListBucketsByIDsQuery, "").HandlerFunc(g.listBucketsByIDsHandler) + return router +} + +func TestGateModular_ListBucketsByIDsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListBucketsByIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListBucketsByIDsQuery, "1,2,3,4") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid id", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListBucketsByIDsQuery, "a,2,3,4") + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid id number", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListBucketsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "repeated id number", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "1,1" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListBucketsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().ListBucketsByIDs(gomock.Any(), gomock.Any(), gomock.Any()).Return(getTestBucketsInIdMap(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + ids := "1" + path := fmt.Sprintf("%s%s/?%s&ids=%s", scheme, testDomain, ListBucketsByIDsQuery, ids) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + assert.Equal(t, "00xF72aDa8130f934887755492879496b026665FbABmock-bucket-name010016997810800xF72aDa8130f934887755492879496b026665FbAB300false00xF72aDa8130f934887755492879496b026665FbAB0x21c349a869bde1f44378936e2a9a15ed3fb2d54a43eaea8787960bba1134cdc20x0cbff0ff3831d61345dbfda5b984e254c4bf87ecf80b45ccbb0635c0547a3b1a12798111699781103", + body) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockListBucketsByIDsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockGetUserGroupsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(getUserGroupsRouterName).Methods(http.MethodGet).Queries(GetUserGroupsQuery, "").HandlerFunc(g.getUserGroupsHandler) + return router +} + +func TestGateModular_GetUserGroupsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserGroups(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?user-groups&start-after=0&limit=1000", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid user address", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?user-groups&start-after=0&limit=1000", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, "invalid_account") + return req + }, + wantedResult: "invalid request header", + }, + { + name: "invalid requestStartAfter", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?user-groups&start-after=a&limit=1000", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid limit", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?user-groups&limit=a", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xmlResponse", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserGroups(gomock.Any(), gomock.Eq(testAccount), gomock.Eq(uint64(2)), gomock.Eq(uint32(1))).Return(getTestGroupMembersResponse(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?user-groups&start-after=2&limit=1", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "0xF72aDa8130f934887755492879496b026665FbABTestGroupName 0000xF72aDa8130f934887755492879496b026665FbAB0xF72aDa8130f934887755492879496b026665FbAB1376086170003219713760861700032197false253402300799", + wantedResultFn: func(body string) bool { + var res types.GfSpGetUserGroupsResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1, len(res.Groups)) + return true + }, + }, + { + name: "large xmlResponse", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserGroups(gomock.Any(), gomock.Eq(testAccount), gomock.Eq(uint64(2)), gomock.Eq(uint32(1000))).Return(getTestGroupMembersResponse(1000), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?user-groups&start-after=2&limit=1000", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpGetUserGroupsResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1000, len(res.Groups)) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetUserGroupsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + begin := time.Now() + router.ServeHTTP(w, tt.request()) + end := time.Now() + log.Printf("GetUserGroupsHandler takes %d to finish", end.UnixMilli()-begin.UnixMilli()) + + assert.Less(t, end.UnixMilli()-begin.UnixMilli(), int64(300)) // we expected this API can return response in 0.3 sec after it gets data from DB. + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockGetGroupMembersHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(getGroupMembersRouterName).Methods(http.MethodGet).Queries(GetGroupMembersQuery, "").HandlerFunc(g.getGroupMembersHandler) + return router +} + +func TestGateModular_GetGroupMembersHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupMembers(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-members&group-id=2&limit=10", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid requestStartAfter", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-members&group-id=2&limit=10&start-after=a", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid group id", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-members&group-id=a&limit=10", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid limit", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-members&group-id=2&limit=a", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xmlResponse", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupMembers(gomock.Any(), gomock.Eq(uint64(73)), gomock.Eq("0x76d32704A1f415a0a8139997Bb40978b9EEf031f"), gomock.Eq(uint32(1))).Return(getTestGroupMembersResponse(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-members&group-id=73&limit=1&start-after=0x76d32704A1f415a0a8139997Bb40978b9EEf031f", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "0xF72aDa8130f934887755492879496b026665FbABTestGroupName 0000xF72aDa8130f934887755492879496b026665FbAB0xF72aDa8130f934887755492879496b026665FbAB1376086170003219713760861700032197false253402300799", + wantedResultFn: func(body string) bool { + var res types.GfSpGetGroupMembersResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1, len(res.Groups)) + return true + }, + }, + { + name: "large xmlResponse", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetGroupMembers(gomock.Any(), gomock.Eq(uint64(73)), gomock.Eq("0x76d32704A1f415a0a8139997Bb40978b9EEf031f"), gomock.Eq(uint32(1000))).Return(getTestGroupMembersResponse(1000), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?group-members&group-id=73&limit=1000&start-after=0x76d32704A1f415a0a8139997Bb40978b9EEf031f", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpGetGroupMembersResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1000, len(res.Groups)) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetGroupMembersHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + begin := time.Now() + router.ServeHTTP(w, tt.request()) + end := time.Now() + log.Printf("GetGroupMembersHandler takes %d to finish", end.UnixMilli()-begin.UnixMilli()) + assert.Less(t, end.UnixMilli()-begin.UnixMilli(), int64(300)) // we expected this API can return response in 0.3 sec after it gets data from DB. + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockGetUserOwnedGroupsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(getUserOwnedGroupsRouterName).Methods(http.MethodGet).Queries(GetUserOwnedGroupsQuery, "").HandlerFunc(g.getUserOwnedGroupsHandler) + return router +} + +func TestGateModular_GetUserOwnedGroupsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserOwnedGroups(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?owned-groups&start-after=15&limit=1", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid user address", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?owned-groups&start-after=15&limit=1", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, "invalid_account") + return req + }, + wantedResult: "invalid request header", + }, + + { + name: "invalid requestStartAfter", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?owned-groups&start-after=a&limit=1", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "invalid limit", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?owned-groups&start-after=15&limit=a", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xmlResponse", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserOwnedGroups(gomock.Any(), gomock.Eq(testAccount), gomock.Eq(uint64(15)), gomock.Eq(uint32(1))).Return(getTestGroupMembersResponse(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?owned-groups&start-after=15&limit=1", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "0xF72aDa8130f934887755492879496b026665FbABTestGroupName 0000xF72aDa8130f934887755492879496b026665FbAB0xF72aDa8130f934887755492879496b026665FbAB1376086170003219713760861700032197false253402300799", + wantedResultFn: func(body string) bool { + var res types.GfSpGetUserOwnedGroupsResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1, len(res.Groups)) + return true + }, + }, + { + name: "large xmlResponse", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserOwnedGroups(gomock.Any(), gomock.Eq(testAccount), gomock.Eq(uint64(15)), gomock.Eq(uint32(1000))).Return(getTestGroupMembersResponse(1000), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?owned-groups&start-after=15&limit=1000", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResultFn: func(body string) bool { + var res types.GfSpGetUserOwnedGroupsResponse + err := xml.Unmarshal([]byte(body), &res) + if err != nil { + return false + } + assert.Equal(t, 1000, len(res.Groups)) + return true + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetUserOwnedGroupsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + begin := time.Now() + router.ServeHTTP(w, tt.request()) + end := time.Now() + log.Printf("GetUserOwnedGroupsHandler takes %d to finish", end.UnixMilli()-begin.UnixMilli()) + assert.Less(t, end.UnixMilli()-begin.UnixMilli(), int64(300)) // we expected this API can return response in 0.3 sec after it gets data from DB. + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockGetBucketMetaHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + var routers []*mux.Router + routers = append(routers, router.Host("{bucket:.+}."+g.domain).Subrouter()) + routers = append(routers, router.PathPrefix("/{bucket}").Subrouter()) + for _, r := range routers { + r.NewRoute().Name(getBucketMetaRouterName).Methods(http.MethodGet).Queries(GetBucketMetaQuery, "").HandlerFunc(g.getBucketMetaHandler) + } + return router +} + +func TestGateModular_GetBucketMetaHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetBucketMeta(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/%s?%s", scheme, testDomain, mockBucketName, GetBucketMetaQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "mock error", + }, + { + name: "wrong BucketName", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/%s?%s", scheme, testDomain, "aa", GetBucketMetaQuery) // aa is an invalid bucket name + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "invalid request params for query", + }, + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetBucketMeta(gomock.Any(), gomock.Any(), gomock.Any()).Return(getTestVGFInfoBuckets(1)[0], getTestStreamRecord(), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/%s?%s", scheme, testDomain, mockBucketName, GetBucketMetaQuery) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + return req + }, + wantedResult: "0xF72aDa8130f934887755492879496b026665FbABmock-bucket-name010016997810800xF72aDa8130f934887755492879496b026665FbAB300false00xF72aDa8130f934887755492879496b026665FbAB0x21c349a869bde1f44378936e2a9a15ed3fb2d54a43eaea8787960bba1134cdc20x0cbff0ff3831d61345dbfda5b984e254c4bf87ecf80b45ccbb0635c0547a3b1a12798111699781103030x26281179b8885F21f95b0a246c8AD70957A95A230xF72aDa8130f934887755492879496b026665FbAB16997809940240000000000000001000000", + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetBucketMetaHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} + +func mockGetUserBucketsHandlerRoute(t *testing.T, g *GateModular) *mux.Router { + t.Helper() + router := mux.NewRouter().SkipClean(true) + router.Path("/").Name(getUserBucketsRouterName).Methods(http.MethodGet).HandlerFunc(g.getUserBucketsHandler) + + return router +} + +func TestGateModular_GetUserBucketsHandler(t *testing.T) { + + cases := []struct { + name string + fn func() *GateModular + request func() *http.Request + wantedResult string + wantedResultFn func(body string) bool + }{ + { + name: "new request context error", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserBuckets(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, mockErr).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "mock error", + }, + { + name: "invalid user address", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, "invalid_account") + return req + }, + wantedResult: "invalid request header", + }, + { + name: "wrong requestIncludeRemoved", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/?&&include-removed=invalid", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "invalid request params for query", + }, + + { + name: "xml response", + fn: func() *GateModular { + g := setup(t) + ctrl := gomock.NewController(t) + clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + clientMock.EXPECT().GetUserBuckets(gomock.Any(), testAccount, gomock.Any()).Return(getTestVGFInfoBuckets(1), nil).Times(1) + g.baseApp.SetGfSpClient(clientMock) + return g + }, + request: func() *http.Request { + path := fmt.Sprintf("%s%s/", scheme, testDomain) + req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + req.Header.Set(GnfdUserAddressHeader, testAccount) + return req + }, + wantedResult: "0xF72aDa8130f934887755492879496b026665FbABmock-bucket-name010016997810800xF72aDa8130f934887755492879496b026665FbAB300false00xF72aDa8130f934887755492879496b026665FbAB0x21c349a869bde1f44378936e2a9a15ed3fb2d54a43eaea8787960bba1134cdc20x0cbff0ff3831d61345dbfda5b984e254c4bf87ecf80b45ccbb0635c0547a3b1a12798111699781103030x26281179b8885F21f95b0a246c8AD70957A95A23", + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + router := mockGetUserBucketsHandlerRoute(t, tt.fn()) + w := httptest.NewRecorder() + router.ServeHTTP(w, tt.request()) + if tt.wantedResult != "" { + assert.Contains(t, w.Body.String(), tt.wantedResult) + } + if tt.wantedResultFn != nil { + assert.True(t, tt.wantedResultFn(w.Body.String())) + } + }) + } +} From 39c3d5159cd0b0af41642cbd6ebeffa3b495c7a8 Mon Sep 17 00:00:00 2001 From: Ruojun Meng Date: Wed, 29 Nov 2023 15:36:56 +0800 Subject: [PATCH 3/6] fix: lint --- modular/gater/metadata_handler.go | 3 +- modular/gater/metadata_handler_test.go | 17 +++--- store/bsdb/prefix_test.go | 76 +++++++++++--------------- 3 files changed, 43 insertions(+), 53 deletions(-) diff --git a/modular/gater/metadata_handler.go b/modular/gater/metadata_handler.go index f90c0783c..f0137e231 100644 --- a/modular/gater/metadata_handler.go +++ b/modular/gater/metadata_handler.go @@ -5,13 +5,14 @@ import ( "encoding/base64" "encoding/json" "encoding/xml" - "github.com/cosmos/gogoproto/jsonpb" "net/http" "net/url" "strconv" "strings" "time" + "github.com/cosmos/gogoproto/jsonpb" + "github.com/ethereum/go-ethereum/common" "github.com/gorilla/mux" diff --git a/modular/gater/metadata_handler_test.go b/modular/gater/metadata_handler_test.go index 648836131..e96c8ceac 100644 --- a/modular/gater/metadata_handler_test.go +++ b/modular/gater/metadata_handler_test.go @@ -1,18 +1,10 @@ package gater import ( - "cosmossdk.io/math" "encoding/base64" "encoding/json" "encoding/xml" "fmt" - "github.com/bnb-chain/greenfield-storage-provider/modular/metadata/types" - payment_types "github.com/bnb-chain/greenfield/x/payment/types" - storage_types "github.com/bnb-chain/greenfield/x/storage/types" - virtual_types "github.com/bnb-chain/greenfield/x/virtualgroup/types" - "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" "log" "math/big" "net/http" @@ -22,6 +14,15 @@ import ( "testing" "time" + "cosmossdk.io/math" + "github.com/bnb-chain/greenfield-storage-provider/modular/metadata/types" + payment_types "github.com/bnb-chain/greenfield/x/payment/types" + storage_types "github.com/bnb-chain/greenfield/x/storage/types" + virtual_types "github.com/bnb-chain/greenfield/x/virtualgroup/types" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + "github.com/bnb-chain/greenfield-storage-provider/base/gfspclient" ) diff --git a/store/bsdb/prefix_test.go b/store/bsdb/prefix_test.go index c9ee16e69..df048369b 100644 --- a/store/bsdb/prefix_test.go +++ b/store/bsdb/prefix_test.go @@ -31,15 +31,13 @@ func TestBsDBImpl_ListObjectsSuccess(t *testing.T) { AddRow("testPath1/", "testPath1/obj1", "obj1", true, false, bucketName, common.HexToHash("1"), "obj1"). AddRow("testPath2/", "testPath2/obj2", "obj2", true, false, bucketName, common.HexToHash("2"), "obj2")) - // Expectations for GetObjectsTableNameByShardNumber - for i := 0; i < ObjectsNumberOfShards; i++ { - mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?,?)", GetObjectsTableNameByShardNumber(i))). - WithArgs(common.HexToHash("1"), common.HexToHash("2")). - WillReturnRows( - sqlmock.NewRows([]string{"object_id", "object_name"}). - AddRow(common.HexToHash("1"), "obj1"). - AddRow(common.HexToHash("2"), "obj2")) - } + // Expectations for GetObjectsTableName + mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?,?)", GetObjectsTableName(bucketName))). + WithArgs(common.HexToHash("1"), common.HexToHash("2")). + WillReturnRows( + sqlmock.NewRows([]string{"object_id", "object_name"}). + AddRow(common.HexToHash("1"), "obj1"). + AddRow(common.HexToHash("2"), "obj2")) res, err := s.ListObjects(bucketName, "", prefix, maxKeys) @@ -67,14 +65,12 @@ func TestBsDBImpl_ListObjectsWithContinuationToken(t *testing.T) { sqlmock.NewRows([]string{"path_name", "full_name", "name", "is_object", "is_folder", "bucket_name", "object_id", "object_name"}). AddRow("testPath2/", "testPath2/obj2", "obj2", true, false, bucketName, common.HexToHash("2"), "obj2")) - // Expectations for GetObjectsTableNameByShardNumber - for i := 0; i < ObjectsNumberOfShards; i++ { - mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableNameByShardNumber(i))). - WithArgs(common.HexToHash("2")). - WillReturnRows( - sqlmock.NewRows([]string{"object_id", "object_name"}). - AddRow(common.HexToHash("2"), "obj2")) - } + // Expectations for GetObjectsTableName + mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableName(bucketName))). + WithArgs(common.HexToHash("2")). + WillReturnRows( + sqlmock.NewRows([]string{"object_id", "object_name"}). + AddRow(common.HexToHash("2"), "obj2")) res, err := s.ListObjects(bucketName, continuationToken, prefix, maxKeys) @@ -97,12 +93,10 @@ func TestBsDBImpl_ListObjectsWithEmptyResult(t *testing.T) { WithArgs(bucketName, prefix). WillReturnRows(sqlmock.NewRows([]string{"path_name", "full_name", "name", "is_object", "is_folder", "bucket_name", "object_id", "object_name"})) - for i := 0; i < ObjectsNumberOfShards; i++ { - mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableNameByShardNumber(i))). - WithArgs(sqlmock.AnyArg()). - WillReturnRows( - sqlmock.NewRows([]string{"object_id", "object_name"})) - } + mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableName(bucketName))). + WithArgs(sqlmock.AnyArg()). + WillReturnRows( + sqlmock.NewRows([]string{"object_id", "object_name"})) res, err := s.ListObjects(bucketName, "", prefix, maxKeys) assert.Nil(t, err) @@ -142,13 +136,11 @@ func TestBsDBImpl_ListObjectsWithPathName(t *testing.T) { sqlmock.NewRows([]string{"path_name", "full_name", "name", "is_object", "is_folder", "bucket_name", "object_id", "object_name"}). AddRow("testPath2/", "testPath2/obj2", "obj2", true, false, bucketName, common.HexToHash("2"), "obj2")) - for i := 0; i < ObjectsNumberOfShards; i++ { - mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableNameByShardNumber(i))). - WithArgs(common.HexToHash("2")). - WillReturnRows( - sqlmock.NewRows([]string{"object_id", "object_name"}). - AddRow(common.HexToHash("2"), "obj2")) - } + mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableName(bucketName))). + WithArgs(common.HexToHash("2")). + WillReturnRows( + sqlmock.NewRows([]string{"object_id", "object_name"}). + AddRow(common.HexToHash("2"), "obj2")) res, err := s.ListObjects(bucketName, "", prefix, maxKeys) assert.Nil(t, err) @@ -178,13 +170,11 @@ func TestBsDBImpl_ListObjectsWithPrefixQuery(t *testing.T) { AddRow("testPath2", "testPath2/obj2", "obj2", true, false, bucketName, common.HexToHash("2"), "obj2")) // Set expectation for the object details query on the respective objects_xx table - for i := 0; i < ObjectsNumberOfShards; i++ { - mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableNameByShardNumber(i))). - WithArgs(common.HexToHash("2")). - WillReturnRows( - sqlmock.NewRows([]string{"object_id", "object_name"}). - AddRow(common.HexToHash("2"), "obj2")) - } + mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableName(bucketName))). + WithArgs(common.HexToHash("2")). + WillReturnRows( + sqlmock.NewRows([]string{"object_id", "object_name"}). + AddRow(common.HexToHash("2"), "obj2")) res, err := s.ListObjects(bucketName, "", prefix, maxKeys) assert.Nil(t, err) @@ -213,13 +203,11 @@ func TestBsDBImpl_ListObjectsWithAllConditions(t *testing.T) { sqlmock.NewRows([]string{"path_name", "full_name", "name", "is_object", "is_folder", "bucket_name", "object_id", "object_name"}). AddRow("testPath2", "testPath2/obj2", "obj2", true, false, bucketName, common.HexToHash("2"), "obj2")) - for i := 0; i < ObjectsNumberOfShards; i++ { - mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableNameByShardNumber(i))). - WithArgs(common.HexToHash("2")). - WillReturnRows( - sqlmock.NewRows([]string{"object_id", "object_name"}). - AddRow(common.HexToHash("2"), "obj2")) - } + mock.ExpectQuery(fmt.Sprintf("SELECT * FROM `%s` WHERE object_id in (?)", GetObjectsTableName(bucketName))). + WithArgs(common.HexToHash("2")). + WillReturnRows( + sqlmock.NewRows([]string{"object_id", "object_name"}). + AddRow(common.HexToHash("2"), "obj2")) res, err := s.ListObjects(bucketName, continuationToken, prefix, maxKeys) assert.Nil(t, err) From bb029ffc425167f7dde9b2a0376030f4383a16e9 Mon Sep 17 00:00:00 2001 From: Ruojun Meng Date: Wed, 29 Nov 2023 16:39:12 +0800 Subject: [PATCH 4/6] fix: minor correction --- modular/gater/metadata_handler.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modular/gater/metadata_handler.go b/modular/gater/metadata_handler.go index f0137e231..a5e7d4d3e 100644 --- a/modular/gater/metadata_handler.go +++ b/modular/gater/metadata_handler.go @@ -278,7 +278,6 @@ func (g *GateModular) listObjectsByBucketNameHandler(w http.ResponseWriter, r *h if format == "json" { respBytes, err = json.Marshal(grpcResponse) - w.Header().Set(ContentTypeHeader, ContentTypeJSONHeaderValue) w.Write(respBytes) return @@ -287,7 +286,7 @@ func (g *GateModular) listObjectsByBucketNameHandler(w http.ResponseWriter, r *h respBytes, err = xml.Marshal((*GfSpListObjectsByBucketNameResponse)(grpcResponse)) if err != nil { - log.CtxErrorw(reqCtx.Context(), "failed to get user buckets", "error", err) + log.CtxErrorw(reqCtx.Context(), "failed to list objects by given bucket name", "error", err) return } @@ -608,11 +607,11 @@ func (m GfSpListObjectsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.Start return err } - for k, v := range m.Objects { - for i, c := range v.ObjectInfo.Checksums { - v.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + for k, o := range m.Objects { + for i, c := range o.ObjectInfo.Checksums { + o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) } - e.Encode(ObjectEntry{Id: k, Value: v}) + e.Encode(ObjectEntry{Id: k, Value: o}) } return e.EncodeToken(start.End()) From ed1bb001257e84a070d8cd67b03d500772c80e1a Mon Sep 17 00:00:00 2001 From: Ruojun Meng Date: Wed, 29 Nov 2023 18:11:03 +0800 Subject: [PATCH 5/6] fix: goimports --- modular/gater/metadata_handler.go | 1 - modular/gater/metadata_handler_test.go | 7 ++++--- store/bsdb/prefix.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modular/gater/metadata_handler.go b/modular/gater/metadata_handler.go index a5e7d4d3e..d11e5d579 100644 --- a/modular/gater/metadata_handler.go +++ b/modular/gater/metadata_handler.go @@ -12,7 +12,6 @@ import ( "time" "github.com/cosmos/gogoproto/jsonpb" - "github.com/ethereum/go-ethereum/common" "github.com/gorilla/mux" diff --git a/modular/gater/metadata_handler_test.go b/modular/gater/metadata_handler_test.go index e96c8ceac..8c669b51f 100644 --- a/modular/gater/metadata_handler_test.go +++ b/modular/gater/metadata_handler_test.go @@ -15,13 +15,14 @@ import ( "time" "cosmossdk.io/math" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + "github.com/bnb-chain/greenfield-storage-provider/modular/metadata/types" payment_types "github.com/bnb-chain/greenfield/x/payment/types" storage_types "github.com/bnb-chain/greenfield/x/storage/types" virtual_types "github.com/bnb-chain/greenfield/x/virtualgroup/types" - "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" "github.com/bnb-chain/greenfield-storage-provider/base/gfspclient" ) diff --git a/store/bsdb/prefix.go b/store/bsdb/prefix.go index e877b57b4..783178bb1 100644 --- a/store/bsdb/prefix.go +++ b/store/bsdb/prefix.go @@ -111,7 +111,7 @@ func (b *BsDBImpl) filterObjects(nodes []*SlashPrefixTreeNode, bucketName string err = b.db.Table(GetObjectsTableName(bucketName)). Where("object_id in (?)", objectIDs). Find(&objects).Error - //stop after finding one set? + if err != nil { return nil, err } From e09073ffc9516c964e074f1d6679d261b87b2845 Mon Sep 17 00:00:00 2001 From: Ruojun Meng Date: Thu, 30 Nov 2023 11:01:46 +0800 Subject: [PATCH 6/6] fix: refine code --- modular/gater/metadata_handler.go | 121 +---------------------- modular/gater/metadata_handler_test.go | 58 +++++------ modular/metadata/types/metadata_types.go | 98 ++++++++++++++++++ 3 files changed, 132 insertions(+), 145 deletions(-) create mode 100644 modular/metadata/types/metadata_types.go diff --git a/modular/gater/metadata_handler.go b/modular/gater/metadata_handler.go index d11e5d579..38e9c8ab9 100644 --- a/modular/gater/metadata_handler.go +++ b/modular/gater/metadata_handler.go @@ -3,7 +3,6 @@ package gater import ( "bytes" "encoding/base64" - "encoding/json" "encoding/xml" "net/http" "net/url" @@ -165,7 +164,6 @@ func (g *GateModular) listObjectsByBucketNameHandler(w http.ResponseWriter, r *h requestDelimiter = queryParams.Get(ListObjectsDelimiterQuery) requestPrefix = queryParams.Get(ListObjectsPrefixQuery) requestIncludeRemoved = queryParams.Get(ListObjectsIncludeRemovedQuery) - format := queryParams.Get("format") if requestDelimiter != "" && requestDelimiter != "/" { log.CtxErrorw(reqCtx.Context(), "failed to check delimiter", "delimiter", requestDelimiter, "error", err) @@ -275,14 +273,7 @@ func (g *GateModular) listObjectsByBucketNameHandler(w http.ResponseWriter, r *h ContinuationToken: continuationToken, } - if format == "json" { - respBytes, err = json.Marshal(grpcResponse) - w.Header().Set(ContentTypeHeader, ContentTypeJSONHeaderValue) - w.Write(respBytes) - return - } - - respBytes, err = xml.Marshal((*GfSpListObjectsByBucketNameResponse)(grpcResponse)) + respBytes, err = xml.Marshal(grpcResponse) if err != nil { log.CtxErrorw(reqCtx.Context(), "failed to list objects by given bucket name", "error", err) @@ -339,7 +330,7 @@ func (g *GateModular) getObjectMetaHandler(w http.ResponseWriter, r *http.Reques grpcResponse := &types.GfSpGetObjectMetaResponse{ Object: resp, } - respBytes, err = xml.Marshal((*GfSpGetObjectMetaResponse)(grpcResponse)) + respBytes, err = xml.Marshal(grpcResponse) if err != nil { log.Errorf("failed to get object meta", "error", err) @@ -589,33 +580,6 @@ func (g *GateModular) getGroupListHandler(w http.ResponseWriter, r *http.Request w.Write(respBytes) } -type GfSpListObjectsByIDsResponse types.GfSpListObjectsByIDsResponse - -type ObjectEntry struct { - Id uint64 - Value *types.Object -} - -func (m GfSpListObjectsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - if len(m.Objects) == 0 { - return nil - } - - err := e.EncodeToken(start) - if err != nil { - return err - } - - for k, o := range m.Objects { - for i, c := range o.ObjectInfo.Checksums { - o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) - } - e.Encode(ObjectEntry{Id: k, Value: o}) - } - - return e.EncodeToken(start.End()) -} - // listObjectsByIDsHandler list objects by object ids func (g *GateModular) listObjectsByIDsHandler(w http.ResponseWriter, r *http.Request) { var ( @@ -686,7 +650,7 @@ func (g *GateModular) listObjectsByIDsHandler(w http.ResponseWriter, r *http.Req } grpcResponse := &types.GfSpListObjectsByIDsResponse{Objects: objects} - respBytes, err = xml.Marshal((*GfSpListObjectsByIDsResponse)(grpcResponse)) + respBytes, err = xml.Marshal(grpcResponse) if err != nil { log.Errorf("failed to list objects by ids", "error", err) return @@ -696,30 +660,6 @@ func (g *GateModular) listObjectsByIDsHandler(w http.ResponseWriter, r *http.Req w.Write(respBytes) } -type GfSpListBucketsByIDsResponse types.GfSpListBucketsByIDsResponse - -type BucketEntry struct { - Id uint64 - Value *types.Bucket -} - -func (m GfSpListBucketsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - if len(m.Buckets) == 0 { - return nil - } - - err := e.EncodeToken(start) - if err != nil { - return err - } - - for k, v := range m.Buckets { - e.Encode(BucketEntry{Id: k, Value: v}) - } - - return e.EncodeToken(start.End()) -} - // listBucketsByIDsHandler list buckets by bucket ids func (g *GateModular) listBucketsByIDsHandler(w http.ResponseWriter, r *http.Request) { var ( @@ -790,7 +730,7 @@ func (g *GateModular) listBucketsByIDsHandler(w http.ResponseWriter, r *http.Req } grpcResponse := &types.GfSpListBucketsByIDsResponse{Buckets: buckets} - respBytes, err = xml.Marshal((*GfSpListBucketsByIDsResponse)(grpcResponse)) + respBytes, err = xml.Marshal(grpcResponse) if err != nil { log.Errorf("failed to list buckets by ids", "error", err) return @@ -2581,57 +2521,6 @@ func (g *GateModular) listUserPaymentAccountsHandler(w http.ResponseWriter, r *h w.Write(respBytes) } -type GfSpListGroupsByIDsResponse types.GfSpListGroupsByIDsResponse - -type GroupEntry struct { - Id uint64 - Value *types.Group -} - -func (m GfSpListGroupsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - if len(m.Groups) == 0 { - return nil - } - - err := e.EncodeToken(start) - if err != nil { - return err - } - - for k, v := range m.Groups { - e.Encode(GroupEntry{Id: k, Value: v}) - } - - return e.EncodeToken(start.End()) -} - -type GfSpListObjectsByBucketNameResponse types.GfSpListObjectsByBucketNameResponse - -func (m GfSpListObjectsByBucketNameResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - type Alias GfSpListObjectsByBucketNameResponse - // Create a new struct with Base64-encoded Checksums field - responseAlias := Alias(m) - for _, o := range responseAlias.Objects { - for i, c := range o.ObjectInfo.Checksums { - o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) - } - } - return e.EncodeElement(responseAlias, start) -} - -type GfSpGetObjectMetaResponse types.GfSpGetObjectMetaResponse - -func (m GfSpGetObjectMetaResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - type Alias GfSpGetObjectMetaResponse - // Create a new struct with Base64-encoded Checksums field - responseAlias := Alias(m) - o := responseAlias.Object - for i, c := range o.ObjectInfo.Checksums { - o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) - } - return e.EncodeElement(responseAlias, start) -} - // listGroupsByIDsHandler list groups by ids func (g *GateModular) listGroupsByIDsHandler(w http.ResponseWriter, r *http.Request) { var ( @@ -2702,7 +2591,7 @@ func (g *GateModular) listGroupsByIDsHandler(w http.ResponseWriter, r *http.Requ } grpcResponse := &types.GfSpListGroupsByIDsResponse{Groups: groups} - respBytes, err = xml.Marshal((*GfSpListGroupsByIDsResponse)(grpcResponse)) + respBytes, err = xml.Marshal(grpcResponse) if err != nil { log.Errorf("failed to list groups by ids", "error", err) return diff --git a/modular/gater/metadata_handler_test.go b/modular/gater/metadata_handler_test.go index 8c669b51f..7cdc902af 100644 --- a/modular/gater/metadata_handler_test.go +++ b/modular/gater/metadata_handler_test.go @@ -2,7 +2,6 @@ package gater import ( "encoding/base64" - "encoding/json" "encoding/xml" "fmt" "log" @@ -472,34 +471,35 @@ func TestGateModular_ListObjectsByBucketNameHandler(t *testing.T) { }, wantedResult: "invalid request params for query", }, - { - name: "json response", - fn: func() *GateModular { - g := setup(t) - ctrl := gomock.NewController(t) - clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) - clientMock.EXPECT().ListObjectsByBucketName(gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(mockData, uint64(0), uint64(0), false, "", "", "", "", nil, "", nil).Times(1) - g.baseApp.SetGfSpClient(clientMock) - return g - }, - request: func() *http.Request { - path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&format=json", scheme, mockBucketName, testDomain) - req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) - return req - }, - wantedResultFn: func(body string) bool { - var res types.GfSpListObjectsByBucketNameResponse - err := json.Unmarshal([]byte(body), &res) - if err != nil { - return false - } - assert.Equal(t, len(mockData), len(res.Objects)) - assert.Equal(t, mockData[0].ObjectInfo.Id, res.Objects[0].ObjectInfo.Id) - return true - }, - }, + // Disable the following json response unit test and will add it back once we add json response support in metadata_handler soon. + //{ + // name: "json response", + // fn: func() *GateModular { + // g := setup(t) + // ctrl := gomock.NewController(t) + // clientMock := gfspclient.NewMockGfSpClientAPI(ctrl) + // clientMock.EXPECT().ListObjectsByBucketName(gomock.Any(), gomock.Any(), gomock.Any(), + // gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + // ).Return(mockData, uint64(0), uint64(0), false, "", "", "", "", nil, "", nil).Times(1) + // g.baseApp.SetGfSpClient(clientMock) + // return g + // }, + // request: func() *http.Request { + // path := fmt.Sprintf("%s%s.%s/?max-keys=1000&delimiter=%%2F&format=json", scheme, mockBucketName, testDomain) + // req := httptest.NewRequest(http.MethodGet, path, strings.NewReader("")) + // return req + // }, + // wantedResultFn: func(body string) bool { + // var res types.GfSpListObjectsByBucketNameResponse + // err := json.Unmarshal([]byte(body), &res) + // if err != nil { + // return false + // } + // assert.Equal(t, len(mockData), len(res.Objects)) + // assert.Equal(t, mockData[0].ObjectInfo.Id, res.Objects[0].ObjectInfo.Id) + // return true + // }, + //}, { name: "xml response", fn: func() *GateModular { diff --git a/modular/metadata/types/metadata_types.go b/modular/metadata/types/metadata_types.go new file mode 100644 index 000000000..e0bd884f1 --- /dev/null +++ b/modular/metadata/types/metadata_types.go @@ -0,0 +1,98 @@ +package types + +import ( + "encoding/base64" + "encoding/xml" +) + +func (m GfSpListObjectsByBucketNameResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type Alias GfSpListObjectsByBucketNameResponse + // Create a new struct with Base64-encoded Checksums field + responseAlias := Alias(m) + for _, o := range responseAlias.Objects { + for i, c := range o.ObjectInfo.Checksums { + o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + } + } + return e.EncodeElement(responseAlias, start) +} + +func (m GfSpGetObjectMetaResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type Alias GfSpGetObjectMetaResponse + // Create a new struct with Base64-encoded Checksums field + responseAlias := Alias(m) + o := responseAlias.Object + for i, c := range o.ObjectInfo.Checksums { + o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + } + return e.EncodeElement(responseAlias, start) +} + +type GroupEntry struct { + Id uint64 + Value *Group +} + +func (m GfSpListGroupsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if len(m.Groups) == 0 { + return nil + } + + err := e.EncodeToken(start) + if err != nil { + return err + } + + for k, v := range m.Groups { + e.Encode(GroupEntry{Id: k, Value: v}) + } + + return e.EncodeToken(start.End()) +} + +type ObjectEntry struct { + Id uint64 + Value *Object +} + +func (m GfSpListObjectsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if len(m.Objects) == 0 { + return nil + } + + err := e.EncodeToken(start) + if err != nil { + return err + } + + for k, o := range m.Objects { + for i, c := range o.ObjectInfo.Checksums { + o.ObjectInfo.Checksums[i] = []byte(base64.StdEncoding.EncodeToString(c)) + } + e.Encode(ObjectEntry{Id: k, Value: o}) + } + + return e.EncodeToken(start.End()) +} + +type BucketEntry struct { + Id uint64 + Value *Bucket +} + +func (m GfSpListBucketsByIDsResponse) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if len(m.Buckets) == 0 { + return nil + } + + err := e.EncodeToken(start) + if err != nil { + return err + } + + for k, v := range m.Buckets { + e.Encode(BucketEntry{Id: k, Value: v}) + } + + return e.EncodeToken(start.End()) +}