From f3e3513a598eea232b3c9bdcc0335a8753b75223 Mon Sep 17 00:00:00 2001 From: krli Date: Mon, 16 May 2022 23:44:25 +0800 Subject: [PATCH] Register code repository file tree api add notes fix tree types Add the function of parsing user information CreateGitRepoFile add authorinfo delete fmt print add commit author info add notes, agg GetUserBasicInfoFromReq errors add userinfo filter add filter unittest fix userinfo filter fix log fix context key type add exception request filter, parallel page tool add userinfo filter fix notes add notes fix jwt unittest fix unittest TestAboutGitRepositoryFileTreeTable --- apis/meta/v1alpha1/createmeta_types.go | 2 + apis/meta/v1alpha1/gitrepofiletree_types.go | 69 ++++++++++ apis/meta/v1alpha1/user_types.go | 47 +++++++ go.mod | 6 +- go.sum | 2 + logging/exception_request_filter.go | 72 +++++++++++ parallel/page.go | 91 +++++++++++++ parallel/page_test.go | 35 +++++ plugin/client/gitcontent.go | 1 + plugin/client/gitfiletree.go | 70 ++++++++++ plugin/client/interface.go | 6 + plugin/client/plugin_client.go | 5 + plugin/route/gitrepofile.go | 1 + plugin/route/gitrepofiletree.go | 83 ++++++++++++ plugin/route/gitrepofiletree_test.go | 134 ++++++++++++++++++++ plugin/route/route.go | 7 + sharedmain/app.go | 8 ++ user/context.go | 42 ++++++ user/filter_test.go | 45 +++++++ user/filters.go | 36 ++++++ user/userinfo.go | 73 +++++++++++ user/userinfo_test.go | 58 +++++++++ 22 files changed, 892 insertions(+), 1 deletion(-) create mode 100644 apis/meta/v1alpha1/gitrepofiletree_types.go create mode 100644 logging/exception_request_filter.go create mode 100644 parallel/page.go create mode 100644 parallel/page_test.go create mode 100644 plugin/client/gitfiletree.go create mode 100644 plugin/route/gitrepofiletree.go create mode 100644 plugin/route/gitrepofiletree_test.go create mode 100644 user/context.go create mode 100644 user/filter_test.go create mode 100644 user/filters.go create mode 100644 user/userinfo.go create mode 100644 user/userinfo_test.go diff --git a/apis/meta/v1alpha1/createmeta_types.go b/apis/meta/v1alpha1/createmeta_types.go index 224dd880d..df0165201 100644 --- a/apis/meta/v1alpha1/createmeta_types.go +++ b/apis/meta/v1alpha1/createmeta_types.go @@ -26,6 +26,8 @@ type CreateRepoFileParams struct { Message string `json:"message"` // Content must be base64 encoded Content []byte `json:"content"` + // Author commit author + Author *GitUserBaseInfo `json:"author,omitempty"` } // CreateRepoFilePayload option for create file and commit + push diff --git a/apis/meta/v1alpha1/gitrepofiletree_types.go b/apis/meta/v1alpha1/gitrepofiletree_types.go new file mode 100644 index 000000000..7cb309c9e --- /dev/null +++ b/apis/meta/v1alpha1/gitrepofiletree_types.go @@ -0,0 +1,69 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + GitRepositoryFileTreeGVK = GroupVersion.WithKind("GitRepositoryFileTree") +) + +// GitRepoFile object for plugins +type GitRepositoryFileTree struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GitRepositoryFileTreeSpec `json:"spec"` +} + +// GitRepoFileSpec spec for repository's file +type GitRepositoryFileTreeSpec struct { + Tree []GitRepositoryFileTreeNode `json:"tree"` +} + +type GitRepositoryFileTreeNodeType string + +const ( + // TreeNodeBlobType represents a file + TreeNodeBlobType GitRepositoryFileTreeNodeType = "blob" + // TreeNodeTreeType represents a folder + TreeNodeTreeType GitRepositoryFileTreeNodeType = "tree" +) + +// GitRepositoryFileTreeNode represents a node in the file system +type GitRepositoryFileTreeNode struct { + // Sha is the ID of the node + Sha string `json:"sha"` + // Name is the name of the node + Name string `json:"name"` + // Path is the path of the node + Path string `json:"path"` + // Type is the type of the node + Type GitRepositoryFileTreeNodeType `json:"type"` + // Mode indicates the permission level of the file + Mode string `json:"mode"` +} + +// Requesting parameters for the File Tree API +type GitRepoFileTreeOption struct { + GitRepo + Path string `json:"path"` + TreeSha string `json:"tree_sha"` + Recursive bool `json:"recursive"` +} diff --git a/apis/meta/v1alpha1/user_types.go b/apis/meta/v1alpha1/user_types.go index d271dbaf1..91876229f 100644 --- a/apis/meta/v1alpha1/user_types.go +++ b/apis/meta/v1alpha1/user_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha1 import ( + "github.com/golang-jwt/jwt" + authenticationv1 "k8s.io/api/authentication/v1" authv1 "k8s.io/api/authorization/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,3 +65,48 @@ func UserResourceAttributes(verb string) authv1.ResourceAttributes { Verb: verb, } } + +// UserInfo is generic user information types +type UserInfo struct { + // UserInfo holds the information about the user needed to implement the user.Info interface. + authenticationv1.UserInfo +} + +const ( + userNameKey string = "name" + userEmailKey string = "email" +) + +// Get information from jwt to populate userinfo +func (user *UserInfo) FromJWT(claims jwt.MapClaims) { + + _username, ok := claims[userNameKey] + if ok { + username := _username.(string) + user.Username = username + } + _email, ok := claims[userEmailKey] + if ok { + email := _email.(string) + user.Extra = make(map[string]authenticationv1.ExtraValue) + user.Extra[userEmailKey] = authenticationv1.ExtraValue{email} + } + return +} + +// get username from UserInfo +func (user *UserInfo) GetName() string { + return user.Username +} + +// get email from UserInfo +func (user *UserInfo) GetEmail() string { + extraValue, ok := user.Extra[userEmailKey] + if !ok { + return "" + } + if len(extraValue) > 0 { + return extraValue[0] + } + return "" +} diff --git a/go.mod b/go.mod index b08505ac4..70c975a58 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,11 @@ require ( go.opentelemetry.io/otel/trace v1.2.0 ) +require ( + github.com/golang-jwt/jwt v3.2.2+incompatible + golang.org/x/net v0.0.0-20210917221730-978cfadd31cf +) + require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.3.0 // indirect @@ -103,7 +108,6 @@ require ( go.uber.org/automaxprocs v1.4.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect - golang.org/x/net v0.0.0-20210917221730-978cfadd31cf // indirect golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect diff --git a/go.sum b/go.sum index c2b1568a1..6eec1d28e 100644 --- a/go.sum +++ b/go.sum @@ -239,6 +239,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/logging/exception_request_filter.go b/logging/exception_request_filter.go new file mode 100644 index 000000000..bb82f8bd9 --- /dev/null +++ b/logging/exception_request_filter.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logging + +import ( + "net/http" + "strings" + + "github.com/emicklei/go-restful/v3" + "knative.dev/pkg/logging" +) + +// Is it an abnormal status code +func isExceptionStatusCode(resp *restful.Response) bool { + if resp.StatusCode() == http.StatusOK || resp.StatusCode() == http.StatusCreated { + return false + } + return true +} + +// ExceptionRequestFilter is used to catch requests with exceptions +func ExceptionRequestFilter(serviceName string, ignorePaths []string, ignoreStatusCodes []int) restful.FilterFunction { + + isIgnoredPath := func(req *restful.Request) bool { + routePath := strings.TrimPrefix(req.SelectedRoutePath(), "/") + for _, _item := range ignorePaths { + item := strings.TrimPrefix(_item, "/") + if item == routePath { + return true + } + } + return false + } + isIgnoredStatusCode := func(resp *restful.Response) bool { + for _, code := range ignoreStatusCodes { + if resp.StatusCode() == code { + return true + } + } + return false + } + return func(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + if isIgnoredPath(req) { + chain.ProcessFilter(req, resp) + return + } + logger := logging.FromContext(req.Request.Context()) + logger.Debugw("received requests", "req", req) + chain.ProcessFilter(req, resp) + if isIgnoredStatusCode(resp) { + return + } + if isExceptionStatusCode(resp) { + logger.Debugw("status code of the response to the exception caught", "code", resp.StatusCode(), "req", req, "resp", resp) + return + } + } +} diff --git a/parallel/page.go b/parallel/page.go new file mode 100644 index 000000000..f9a710408 --- /dev/null +++ b/parallel/page.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parallel + +import ( + "context" + "fmt" + "time" + + "k8s.io/utils/trace" + "knative.dev/pkg/logging" +) + +// PageRequestFunc is a tool for concurrent processing of pagination +type PageRequestFunc struct { + // RequestPage for concurrent request paging + RequestPage func(ctx context.Context, pageSize int, page int) (interface{}, error) + // PageResult for get paging information + PageResult func(items interface{}) (total int, currentPageLen int, err error) +} + +// Concurrent request paging +func PageRequest(ctx context.Context, logName string, concurrency int, pageSize int, f PageRequestFunc) ([]interface{}, error) { + log := trace.New("PageRequest", trace.Field{Key: "name", Value: logName}) + logger := logging.FromContext(ctx) + + defer func() { + log.LogIfLong(3 * time.Second) + }() + + items, err := f.RequestPage(ctx, pageSize, 1) + if err != nil { + return nil, err + } + log.Step("requested page 1") + total, firstPageLen, err := f.PageResult(items) + if err != nil { + return nil, err + } + if firstPageLen < pageSize { + return []interface{}{items}, nil + } + + if total == firstPageLen { + return []interface{}{items}, nil + } + + var request = func(i int) func() (interface{}, error) { + return func() (interface{}, error) { + items, err := f.RequestPage(ctx, pageSize, i) + log.Step(fmt.Sprintf("requested page %d", i)) + return items, err + } + } + + totalPage := total / pageSize + if total%pageSize != 0 { + totalPage = totalPage + 1 + } + + if totalPage-1 < concurrency { // first page we have requested, so skip first page + concurrency = totalPage - 1 + } + + p := P(logger, "PageRequest").FailFast().SetConcurrent(concurrency).Context(ctx) + for i := 2; i <= totalPage; i++ { + p.Add(request(i)) + } + + results, err := p.Do().Wait() + if err != nil { + return nil, err + } + + return append([]interface{}{items}, results...), nil + +} diff --git a/parallel/page_test.go b/parallel/page_test.go new file mode 100644 index 000000000..7be3cd65b --- /dev/null +++ b/parallel/page_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parallel + +import ( + "context" + "fmt" + "testing" +) + +func TestPage(t *testing.T) { + PageRequest(context.Background(), "TestPageResult", 2, 10, PageRequestFunc{ + RequestPage: func(ctx context.Context, pageSize int, page int) (interface{}, error) { + fmt.Printf("request -> page: %d, pagesize: %d\n", page, pageSize) + return nil, nil + }, + PageResult: func(items interface{}) (total int, currentPageLen int, err error) { + return 8, 6, nil + }, + }) +} diff --git a/plugin/client/gitcontent.go b/plugin/client/gitcontent.go index 8ffca33c2..9f7633a05 100644 --- a/plugin/client/gitcontent.go +++ b/plugin/client/gitcontent.go @@ -78,6 +78,7 @@ func (g *gitContent) Create(ctx context.Context, baseURL *duckv1.Addressable, pa } w.Close() payload.Content = b.Bytes() + options = append(options, MetaOpts(g.meta), SecretOpts(g.secret), BodyOpts(payload.CreateRepoFileParams), ResultOpts(commitInfo)) if payload.Repository == "" { return nil, errors.NewBadRequest("repo is empty string") diff --git a/plugin/client/gitfiletree.go b/plugin/client/gitfiletree.go new file mode 100644 index 000000000..83b232fc2 --- /dev/null +++ b/plugin/client/gitfiletree.go @@ -0,0 +1,70 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + + metav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1" + duckv1 "knative.dev/pkg/apis/duck/v1" +) + +// ClientGitRepositoryFileTree defines the request interface for the file tree +type ClientGitRepositoryFileTree interface { + GetGitRepositoryFileTree(ctx context.Context, baseURL *duckv1.Addressable, option metav1alpha1.GitRepoFileTreeOption, options ...OptionFunc) (*metav1alpha1.GitRepositoryFileTree, error) +} + +type gitRepositoryFileTree struct { + client Client + meta Meta + secret corev1.Secret +} + +// init gitRepositoryFileTree +func newGitRepositoryFileTree(client Client, meta Meta, secret corev1.Secret) ClientGitRepositoryFileTree { + return &gitRepositoryFileTree{ + client: client, + meta: meta, + secret: secret, + } +} + +// GetGitRepositoryFileTree call the integrations api +func (g *gitRepositoryFileTree) GetGitRepositoryFileTree(ctx context.Context, baseURL *duckv1.Addressable, option metav1alpha1.GitRepoFileTreeOption, options ...OptionFunc) (*metav1alpha1.GitRepositoryFileTree, error) { + fileTree := &metav1alpha1.GitRepositoryFileTree{} + var recursiveValue string + if option.Recursive { + recursiveValue = "true" + } else { + recursiveValue = "false" + } + options = append(options, MetaOpts(g.meta), SecretOpts(g.secret), QueryOpts(map[string]string{"path": option.Path, "tree_sha": option.TreeSha, "recursive": recursiveValue}), ResultOpts(fileTree)) + if option.Repository == "" { + return nil, errors.NewBadRequest("repo is empty string") + } else if option.Path == "" { + return nil, errors.NewBadRequest("file path is empty string") + } + uri := fmt.Sprintf("projects/%s/coderepositories/%s/tree", option.Project, handlePathParamHasSlash(option.Repository)) + if err := g.client.Get(ctx, baseURL, uri, options...); err != nil { + return nil, err + } + return fileTree, nil +} diff --git a/plugin/client/interface.go b/plugin/client/interface.go index 2527e991d..f10361b1e 100644 --- a/plugin/client/interface.go +++ b/plugin/client/interface.go @@ -251,6 +251,12 @@ type GitRepositoryGetter interface { GetGitRepository(ctx context.Context, repoOption metav1alpha1.GitRepo) (metav1alpha1.GitRepository, error) } +// GitRepositoryFileTreeGetter get git repository file tree +type GitRepositoryFileTreeGetter interface { + Interface + GetGitRepositoryFileTree(ctx context.Context, repoOption metav1alpha1.GitRepoFileTreeOption, listOption metav1alpha1.ListOptions) (metav1alpha1.GitRepositoryFileTree, error) +} + // GitCommitStatusLister list git commit status type GitCommitStatusLister interface { Interface diff --git a/plugin/client/plugin_client.go b/plugin/client/plugin_client.go index 66635c28a..1ec33cf86 100644 --- a/plugin/client/plugin_client.go +++ b/plugin/client/plugin_client.go @@ -226,6 +226,11 @@ func (p *PluginClient) GitRepository(meta Meta, secret corev1.Secret) ClientGitR return newGitRepository(p, meta, secret) } +// GitRepositoryFileTree get repo file tree client +func (p *PluginClient) GitRepositoryFileTree(meta Meta, secret corev1.Secret) ClientGitRepositoryFileTree { + return newGitRepositoryFileTree(p, meta, secret) +} + // GitCommitComment get commit comment client func (p *PluginClient) GitCommitComment(meta Meta, secret corev1.Secret) ClientGitCommitComment { return newGitCommitComment(p, meta, secret) diff --git a/plugin/route/gitrepofile.go b/plugin/route/gitrepofile.go index 0d35fe6c1..c71c776f3 100644 --- a/plugin/route/gitrepofile.go +++ b/plugin/route/gitrepofile.go @@ -122,6 +122,7 @@ func (a *gitRepoFileCreator) CreateGitRepoFile(request *restful.Request, respons kerrors.HandleError(request, response, err) return } + payload := metav1alpha1.CreateRepoFilePayload{ GitRepo: metav1alpha1.GitRepo{Repository: repo, Project: project}, CreateRepoFileParams: params, diff --git a/plugin/route/gitrepofiletree.go b/plugin/route/gitrepofiletree.go new file mode 100644 index 000000000..ea7fb7d05 --- /dev/null +++ b/plugin/route/gitrepofiletree.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package route + +import ( + "net/http" + + kerrors "github.com/katanomi/pkg/errors" + + restfulspec "github.com/emicklei/go-restful-openapi/v2" + "github.com/emicklei/go-restful/v3" + metav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1" + "github.com/katanomi/pkg/plugin/client" +) + +// gitRepositoryFileTreeGetter is impl ClientGitRepositoryFileTree interface +type gitRepositoryFileTreeGetter struct { + impl client.GitRepositoryFileTreeGetter + tags []string +} + +// NewGitRepositoryFileTreeGetter get a git repo route with plugin client +func NewGitRepositoryFileTreeGetter(impl client.GitRepositoryFileTreeGetter) Route { + return &gitRepositoryFileTreeGetter{ + tags: []string{"git", "repositories", "tree"}, + impl: impl, + } +} + +// Register route +func (g *gitRepositoryFileTreeGetter) Register(ws *restful.WebService) { + repositoryParam := ws.PathParameter("repository", "file belong to repository") + projectParam := ws.PathParameter("project", "repository belong to project") + pathParam := ws.QueryParameter("path", "file path") + treeShaParam := ws.QueryParameter("tree_sha", "sha for file tree") + recursive := ws.QueryParameter("recursive", "recursive switch") + ws.Route( + ws.GET("/projects/{project:*}/coderepositories/{repository}/tree").To(g.GetGitRepositoryFileTree). + Doc("GetGitRepositoryFileTree").Param(projectParam).Param(repositoryParam).Param(pathParam).Param(treeShaParam).Param(recursive). + Metadata(restfulspec.KeyOpenAPITags, g.tags). + Returns(http.StatusOK, "OK", metav1alpha1.GitRepositoryFileTree{}), + ) + return +} + +// GetGitRepositoryFileTree get repo file tree +func (g *gitRepositoryFileTreeGetter) GetGitRepositoryFileTree(request *restful.Request, response *restful.Response) { + repo := handlePathParamHasSlash(request.PathParameter("repository")) + project := request.PathParameter("project") + path := request.QueryParameter("path") + recursive := request.QueryParameter("recursive") + recursiveValue := recursive == "true" + treeSha := request.QueryParameter("tree_sha") + + ctx := request.Request.Context() + option := metav1alpha1.GitRepoFileTreeOption{ + GitRepo: metav1alpha1.GitRepo{Repository: repo, Project: project}, + Path: path, + TreeSha: treeSha, + Recursive: recursiveValue, + } + listOption := metav1alpha1.ListOptions{} + fileTree, err := g.impl.GetGitRepositoryFileTree(ctx, option, listOption) + if err != nil { + kerrors.HandleError(request, response, err) + return + } + response.WriteHeaderAndEntity(http.StatusOK, fileTree) +} diff --git a/plugin/route/gitrepofiletree_test.go b/plugin/route/gitrepofiletree_test.go new file mode 100644 index 000000000..97be2125b --- /dev/null +++ b/plugin/route/gitrepofiletree_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2021 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package route_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + + "net/http" + "net/http/httptest" + "testing" + + "github.com/emicklei/go-restful/v3" + kerrors "github.com/katanomi/pkg/errors" + "github.com/katanomi/pkg/plugin/client" + "github.com/katanomi/pkg/plugin/route" + "k8s.io/apimachinery/pkg/api/errors" + + metav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1" + . "github.com/onsi/gomega" + "go.uber.org/zap" + authv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestAboutGitRepositoryFileTreeTable(t *testing.T) { + g := NewGomegaWithT(t) + + var meta string + var c []byte + var container *restful.Container + var err error + var ws *restful.WebService + var httpRequest *http.Request + var httpWriter *httptest.ResponseRecorder + var metaData client.Meta + var data []byte + var FilePath string + var params metav1alpha1.GitRepoFileTreeOption + + metaData = client.Meta{BaseURL: "http://api.test", Version: "v1"} + data, _ = json.Marshal(metaData) + FilePath = ".build%/build.yaml" + params = metav1alpha1.GitRepoFileTreeOption{ + Path: FilePath, + TreeSha: "master", + Recursive: false, + } + meta = base64.StdEncoding.EncodeToString(data) + c, _ = json.Marshal(params) + httpRequest, _ = http.NewRequest("GET", "/plugins/v1alpha1/test-by/projects/1/coderepositories/1/tree", bytes.NewBuffer(c)) + httpRequest.Header.Set("content-type", "application/json") + httpRequest.Header.Set(client.PluginMetaHeader, meta) + resourceAtt := authv1.ResourceAttributes{} + + getters := []TestGitRepoFileTreeMockGetter{ + { + MockResult: metav1alpha1.GitRepositoryFileTree{ + TypeMeta: metav1.TypeMeta{ + Kind: metav1alpha1.GitRepositoryFileTreeGVK.Kind, + }}, + }, { + MockError: errors.NewNotFound(schema.GroupResource{}, "error"), + }, { + MockError: errors.NewInternalError(fmt.Errorf("error")), + }, { + MockError: errors.NewTimeoutError("timeout", 0), + }, { + MockError: errors.NewForbidden(schema.GroupResource{ + Group: resourceAtt.Group, + Resource: resourceAtt.Resource}, + resourceAtt.Name, fmt.Errorf("access not allowed")), + }, { + MockError: errors.NewUnauthorized(""), + }, + } + for _, getter := range getters { + ws, err = route.NewService(&getter, client.MetaFilter) + g.Expect(err).To(BeNil()) + container = restful.NewContainer() + container.Add(ws) + + httpWriter = httptest.NewRecorder() + g.Expect(container).NotTo(BeNil()) + container.Dispatch(httpWriter, httpRequest) + if getter.MockError != nil { + g.Expect(container).NotTo(BeNil()) + g.Expect(httpWriter.Code).To(Equal(kerrors.AsStatusCode(getter.MockError))) + } else { + g.Expect(httpWriter.Code).To(Equal(http.StatusOK)) + fileTree := metav1alpha1.GitRepositoryFileTree{} + err := json.Unmarshal(httpWriter.Body.Bytes(), &fileTree) + g.Expect(err).To(BeNil()) + g.Expect(fileTree.Kind).To(Equal(metav1alpha1.GitRepositoryFileTreeGVK.Kind)) + } + } + +} + +// mock ClientGitRepositoryFileTree interface +type TestGitRepoFileTreeMockGetter struct { + MockResult metav1alpha1.GitRepositoryFileTree + MockError error +} + +func (t *TestGitRepoFileTreeMockGetter) Path() string { + return "test-by" +} + +func (t *TestGitRepoFileTreeMockGetter) Setup(_ context.Context, _ *zap.SugaredLogger) error { + return nil +} + +func (t *TestGitRepoFileTreeMockGetter) GetGitRepositoryFileTree(ctx context.Context, repoOption metav1alpha1.GitRepoFileTreeOption, listOption metav1alpha1.ListOptions) (metav1alpha1.GitRepositoryFileTree, error) { + return t.MockResult, t.MockError +} diff --git a/plugin/route/route.go b/plugin/route/route.go index aafc4bc3a..cbab5ae37 100644 --- a/plugin/route/route.go +++ b/plugin/route/route.go @@ -168,6 +168,10 @@ func match(c client.Interface) []Route { routes = append(routes, NewBlobStoreLister(v)) } + if v, ok := c.(client.GitRepositoryFileTreeGetter); ok { + routes = append(routes, NewGitRepositoryFileTreeGetter(v)) + } + authCheck, ok := c.(client.AuthChecker) // uses a default implementation that returns an Unknown allowed result // with an NotImplemented reason @@ -284,6 +288,9 @@ func GetMethods(c client.Interface) []string { if _, ok := c.(client.GitRepositoryGetter); ok { methods = append(methods, "GetGitRepository") } + if _, ok := c.(client.GitRepositoryFileTreeGetter); ok { + methods = append(methods, "GetGitRepositoryFileTree") + } if _, ok := c.(client.GitCommitCommentLister); ok { methods = append(methods, "ListGitCommitComment") } diff --git a/sharedmain/app.go b/sharedmain/app.go index e77667dc8..b7cabaec3 100644 --- a/sharedmain/app.go +++ b/sharedmain/app.go @@ -252,6 +252,14 @@ func (a *AppBuilder) Tracing(ops ...tracing.TraceOption) *AppBuilder { return a } +// DebugExceptionRequests are used to catch exception requests, and print +func (a *AppBuilder) DebugExceptionRequests(ignoreStatusCodes ...int) *AppBuilder { + a.init() + ignorepaths := []string{healthzRoutePath, readyzRoutePath} + a.filters = append(a.filters, klogging.ExceptionRequestFilter(a.Name, ignorepaths, ignoreStatusCodes)) + return a +} + // Log adds logging and logger to the app func (a *AppBuilder) Log() *AppBuilder { a.init() diff --git a/user/context.go b/user/context.go new file mode 100644 index 000000000..2a33fdc1b --- /dev/null +++ b/user/context.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "context" + + metav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1" +) + +type userInfoKey struct{} + +// WithUserInfo returns a copy of parent in which the userinfo value is set +func WithUserInfo(parent context.Context, userinfo metav1alpha1.UserInfo) context.Context { + return context.WithValue(parent, userInfoKey{}, userinfo) +} + +// UserInfoFrom returns the value of the userinfo key on the ctx +func UserInfoFrom(ctx context.Context) (metav1alpha1.UserInfo, bool) { + userinfo, ok := ctx.Value(userInfoKey{}).(metav1alpha1.UserInfo) + return userinfo, ok +} + +// UserInfoValue returns the value of the userinfo key on the ctx, or the empty string if none +func UserInfoValue(ctx context.Context) (result metav1alpha1.UserInfo) { + userinfo, _ := UserInfoFrom(ctx) + return userinfo +} diff --git a/user/filter_test.go b/user/filter_test.go new file mode 100644 index 000000000..4e3d9c170 --- /dev/null +++ b/user/filter_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "net/http" + "testing" + + restful "github.com/emicklei/go-restful/v3" +) + +func TestUserInfoFilter(t *testing.T) { + tokenString := "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImVtYWlsIjoiam9obkB0ZXN0LmNvbSIsImlhdCI6MTY1MzM3NzM0MCwiZXhwIjoxNjUzMzgwOTQwfQ.tT5YWrFu2F_0NWg3JFbpIaC-HbaBgJtRcff_O3Ti225RZztON1gpZOfFf06EIzWolkUrtAfrTFXonHx2rh5w8mK82kFX5sLPxiV3yDfvxU9KuE66IW_48ykFU6puBcnVMZNWUtPLHcH4OAq51zbca9eh3AFiFldnJ3YRJ85Bigky8nc1uf3CCm5c9d7P8q5SGDJZvGLHOXqQ4_aSfpX0HDTHbgw_82d7FpckeDYRdFN89LORaLRChbBM1IdfH4JvI9WtO3PDU7Ce49nMmmidhsESAalxfMrN3LIPmMz7vY0FJaBW24oA0FwJvq1Q6O_jzupMHRGS-clkUImRw185cA" + _req := &http.Request{ + Header: map[string][]string{}, + } + _req.Header.Set("Authorization", tokenString) + + req := &restful.Request{Request: _req} + res := &restful.Response{} + target := func(req *restful.Request, resp *restful.Response) {} + chain := &restful.FilterChain{Target: target} + UserInfoFilter(req, res, chain) + userinfo := UserInfoValue(req.Request.Context()) + if userinfo.GetName() != "John Doe" { + t.Errorf("should name is John Doe, but got %s", userinfo.GetName()) + } + if userinfo.GetEmail() != "john@test.com" { + t.Errorf("should email is john@test.com, but got %s", userinfo.GetEmail()) + } +} diff --git a/user/filters.go b/user/filters.go new file mode 100644 index 000000000..f452810ea --- /dev/null +++ b/user/filters.go @@ -0,0 +1,36 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + restful "github.com/emicklei/go-restful/v3" + "knative.dev/pkg/logging" +) + +// UserInfoFilter is to parse the user login information from the request header into userinfo, and store it in the context +func UserInfoFilter(req *restful.Request, res *restful.Response, chain *restful.FilterChain) { + curContext := req.Request.Context() + log := logging.FromContext(curContext) + userinfo, err := getUserInfoFromReq(req) + if err != nil { + log.Errorw("get user info from request failed", "err", err) + return + } + newCtx := WithUserInfo(curContext, userinfo) + req.Request = req.Request.WithContext(newCtx) + chain.ProcessFilter(req, res) +} diff --git a/user/userinfo.go b/user/userinfo.go new file mode 100644 index 000000000..790da9c54 --- /dev/null +++ b/user/userinfo.go @@ -0,0 +1,73 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "fmt" + "strings" + + "github.com/emicklei/go-restful/v3" + "github.com/golang-jwt/jwt" + metav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1" + "github.com/katanomi/pkg/client" + "k8s.io/apimachinery/pkg/util/errors" +) + +var hmacSampleSecret []byte + +// Parsing the jwt information in the request header +func parseReqjwtToClaims(req *restful.Request) (claims jwt.MapClaims, err error) { + + tokenString := req.HeaderParameter(client.AuthorizationHeader) + ignoreSigningError := "unexpected signing method: RS256" + + if !strings.HasPrefix(tokenString, client.BearerPrefix) { + err = fmt.Errorf("token string not has prefix %s", client.BearerPrefix) + return + } + tokenString = tokenString[len(client.BearerPrefix):] + token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return hmacSampleSecret, nil + }) + if err != nil { + if err.Error() != ignoreSigningError { + // The unexpected signing method test fails and does not affect the parsing + // This conclusion is reliable + return claims, err + } + } + if claims, ok := token.Claims.(jwt.MapClaims); ok { + return claims, nil + } else { + return jwt.MapClaims{}, fmt.Errorf("token.Claims is not jwt.MapClaims type") + } +} + +// Get the userinfo from the request object +func getUserInfoFromReq(req *restful.Request) (userinfo metav1alpha1.UserInfo, err error) { + errorList := make([]error, 0) + claims, err := parseReqjwtToClaims(req) + if err != nil { + return userinfo, err + } + userinfo = metav1alpha1.UserInfo{} + userinfo.FromJWT(claims) + return userinfo, errors.NewAggregate(errorList) +} diff --git a/user/userinfo_test.go b/user/userinfo_test.go new file mode 100644 index 000000000..4297b4ae9 --- /dev/null +++ b/user/userinfo_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package user + +import ( + "net/http" + "testing" + + "github.com/emicklei/go-restful/v3" +) + +func TestGetUserInfo(t *testing.T) { + + _req := &http.Request{ + Header: map[string][]string{}, + } + req := &restful.Request{Request: _req} + + t.Run("normal", func(t *testing.T) { + // placeholder data generated at https://token.dev/ + tokenString := "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImVtYWlsIjoiam9obkB0ZXN0LmNvbSIsImlhdCI6MTY1MzM3NzM0MCwiZXhwIjoxNjUzMzgwOTQwfQ.tT5YWrFu2F_0NWg3JFbpIaC-HbaBgJtRcff_O3Ti225RZztON1gpZOfFf06EIzWolkUrtAfrTFXonHx2rh5w8mK82kFX5sLPxiV3yDfvxU9KuE66IW_48ykFU6puBcnVMZNWUtPLHcH4OAq51zbca9eh3AFiFldnJ3YRJ85Bigky8nc1uf3CCm5c9d7P8q5SGDJZvGLHOXqQ4_aSfpX0HDTHbgw_82d7FpckeDYRdFN89LORaLRChbBM1IdfH4JvI9WtO3PDU7Ce49nMmmidhsESAalxfMrN3LIPmMz7vY0FJaBW24oA0FwJvq1Q6O_jzupMHRGS-clkUImRw185cA" + + req.Request.Header.Set("Authorization", tokenString) + userinfo, _ := getUserInfoFromReq(req) + if userinfo.GetName() != "John Doe" { + t.Errorf("should have John Doe") + } + if userinfo.GetEmail() != "john@test.com" { + t.Errorf("should have test@test.com but got %s", userinfo.GetEmail()) + } + }) + t.Run("not found jwt", func(t *testing.T) { + tokenString := "" + req.Request.Header.Set("Authorization", tokenString) + userinfo, _ := getUserInfoFromReq(req) + if userinfo.GetName() != "" { + t.Errorf("should have nil string") + } + if userinfo.GetEmail() != "" { + t.Errorf("should have nil string but got %s", userinfo.GetEmail()) + } + }) + +}