Register code repository file tree api

add notes

fix tree types

Add the function of parsing user information

CreateGitRepoFile add authorinfo

delete fmt print

add commit author info

add notes, agg GetUserBasicInfoFromReq errors

add userinfo filter

add filter unittest

fix userinfo filter

fix log

fix context key type

add exception request filter, parallel page tool

add userinfo filter

fix notes

add notes

fix jwt unittest

fix unittest TestAboutGitRepositoryFileTreeTable

apis/meta/v1alpha1/createmeta_types.go

@@ -26,6 +26,8 @@ type CreateRepoFileParams struct {
 	Message string `json:"message"`
 	// Content must be base64 encoded
 	Content []byte `json:"content"`
+	// Author commit author
+	Author *GitUserBaseInfo `json:"author,omitempty"`
 }
 
 // CreateRepoFilePayload option for create file and commit + push

apis/meta/v1alpha1/gitrepofiletree_types.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2022 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	GitRepositoryFileTreeGVK = GroupVersion.WithKind("GitRepositoryFileTree")
+)
+
+// GitRepoFile object for plugins
+type GitRepositoryFileTree struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec GitRepositoryFileTreeSpec `json:"spec"`
+}
+
+// GitRepoFileSpec spec for repository's file
+type GitRepositoryFileTreeSpec struct {
+	Tree []GitRepositoryFileTreeNode `json:"tree"`
+}
+
+type GitRepositoryFileTreeNodeType string
+
+const (
+	// TreeNodeBlobType represents a file
+	TreeNodeBlobType GitRepositoryFileTreeNodeType = "blob"
+	// TreeNodeTreeType represents a folder
+	TreeNodeTreeType GitRepositoryFileTreeNodeType = "tree"
+)
+
+// GitRepositoryFileTreeNode represents a node in the file system
+type GitRepositoryFileTreeNode struct {
+	// Sha is the ID of the node
+	Sha string `json:"sha"`
+	// Name is the name of the node
+	Name string `json:"name"`
+	// Path is the path of the node
+	Path string `json:"path"`
+	// Type is the type of the node
+	Type GitRepositoryFileTreeNodeType `json:"type"`
+	// Mode indicates the permission level of the file
+	Mode string `json:"mode"`
+}
+
+// Requesting parameters for the File Tree API
+type GitRepoFileTreeOption struct {
+	GitRepo
+	Path      string `json:"path"`
+	TreeSha   string `json:"tree_sha"`
+	Recursive bool   `json:"recursive"`
+}

apis/meta/v1alpha1/user_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"github.com/golang-jwt/jwt"
+	authenticationv1 "k8s.io/api/authentication/v1"
 	authv1 "k8s.io/api/authorization/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -63,3 +65,48 @@ func UserResourceAttributes(verb string) authv1.ResourceAttributes {
 		Verb:     verb,
 	}
 }
+
+// UserInfo is generic user information types
+type UserInfo struct {
+	// UserInfo holds the information about the user needed to implement the user.Info interface.
+	authenticationv1.UserInfo
+}
+
+const (
+	userNameKey  string = "name"
+	userEmailKey string = "email"
+)
+
+// Get information from jwt to populate userinfo
+func (user *UserInfo) FromJWT(claims jwt.MapClaims) {
+
+	_username, ok := claims[userNameKey]
+	if ok {
+		username := _username.(string)
+		user.Username = username
+	}
+	_email, ok := claims[userEmailKey]
+	if ok {
+		email := _email.(string)
+		user.Extra = make(map[string]authenticationv1.ExtraValue)
+		user.Extra[userEmailKey] = authenticationv1.ExtraValue{email}
+	}
+	return
+}
+
+// get username from UserInfo
+func (user *UserInfo) GetName() string {
+	return user.Username
+}
+
+// get email from UserInfo
+func (user *UserInfo) GetEmail() string {
+	extraValue, ok := user.Extra[userEmailKey]
+	if !ok {
+		return ""
+	}
+	if len(extraValue) > 0 {
+		return extraValue[0]
+	}
+	return ""
+}

go.mod

@@ -52,6 +52,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.2.0
 )
 
+require (
+	github.com/golang-jwt/jwt v3.2.2+incompatible
+	golang.org/x/net v0.0.0-20210917221730-978cfadd31cf
+)
+
 require (
 	contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect
 	contrib.go.opencensus.io/exporter/prometheus v0.3.0 // indirect
@@ -103,7 +108,6 @@ require (
 	go.uber.org/automaxprocs v1.4.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect
-	golang.org/x/net v0.0.0-20210917221730-978cfadd31cf // indirect
 	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect

go.sum

@@ -239,6 +239,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

logging/exception_request_filter.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2022 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logging
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/emicklei/go-restful/v3"
+	"knative.dev/pkg/logging"
+)
+
+// Is it an abnormal status code
+func isExceptionStatusCode(resp *restful.Response) bool {
+	if resp.StatusCode() == http.StatusOK || resp.StatusCode() == http.StatusCreated {
+		return false
+	}
+	return true
+}
+
+// ExceptionRequestFilter is used to catch requests with exceptions
+func ExceptionRequestFilter(serviceName string, ignorePaths []string, ignoreStatusCodes []int) restful.FilterFunction {
+
+	isIgnoredPath := func(req *restful.Request) bool {
+		routePath := strings.TrimPrefix(req.SelectedRoutePath(), "/")
+		for _, _item := range ignorePaths {
+			item := strings.TrimPrefix(_item, "/")
+			if item == routePath {
+				return true
+			}
+		}
+		return false
+	}
+	isIgnoredStatusCode := func(resp *restful.Response) bool {
+		for _, code := range ignoreStatusCodes {
+			if resp.StatusCode() == code {
+				return true
+			}
+		}
+		return false
+	}
+	return func(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
+		if isIgnoredPath(req) {
+			chain.ProcessFilter(req, resp)
+			return
+		}
+		logger := logging.FromContext(req.Request.Context())
+		logger.Debugw("received requests", "req", req)
+		chain.ProcessFilter(req, resp)
+		if isIgnoredStatusCode(resp) {
+			return
+		}
+		if isExceptionStatusCode(resp) {
+			logger.Debugw("status code of the response to the exception caught", "code", resp.StatusCode(), "req", req, "resp", resp)
+			return
+		}
+	}
+}

parallel/page.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2022 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package parallel
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/utils/trace"
+	"knative.dev/pkg/logging"
+)
+
+// PageRequestFunc is a tool for concurrent processing of pagination
+type PageRequestFunc struct {
+	// RequestPage for concurrent request paging
+	RequestPage func(ctx context.Context, pageSize int, page int) (interface{}, error)
+	// PageResult for get paging information
+	PageResult func(items interface{}) (total int, currentPageLen int, err error)
+}
+
+// Concurrent request paging
+func PageRequest(ctx context.Context, logName string, concurrency int, pageSize int, f PageRequestFunc) ([]interface{}, error) {
+	log := trace.New("PageRequest", trace.Field{Key: "name", Value: logName})
+	logger := logging.FromContext(ctx)
+
+	defer func() {
+		log.LogIfLong(3 * time.Second)
+	}()
+
+	items, err := f.RequestPage(ctx, pageSize, 1)
+	if err != nil {
+		return nil, err
+	}
+	log.Step("requested page 1")
+	total, firstPageLen, err := f.PageResult(items)
+	if err != nil {
+		return nil, err
+	}
+	if firstPageLen < pageSize {
+		return []interface{}{items}, nil
+	}
+
+	if total == firstPageLen {
+		return []interface{}{items}, nil
+	}
+
+	var request = func(i int) func() (interface{}, error) {
+		return func() (interface{}, error) {
+			items, err := f.RequestPage(ctx, pageSize, i)
+			log.Step(fmt.Sprintf("requested page %d", i))
+			return items, err
+		}
+	}
+
+	totalPage := total / pageSize
+	if total%pageSize != 0 {
+		totalPage = totalPage + 1
+	}
+
+	if totalPage-1 < concurrency { // first page we have requested, so skip first page
+		concurrency = totalPage - 1
+	}
+
+	p := P(logger, "PageRequest").FailFast().SetConcurrent(concurrency).Context(ctx)
+	for i := 2; i <= totalPage; i++ {
+		p.Add(request(i))
+	}
+
+	results, err := p.Do().Wait()
+	if err != nil {
+		return nil, err
+	}
+
+	return append([]interface{}{items}, results...), nil
+
+}

parallel/page_test.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2022 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package parallel
+
+import (
+	"context"
+	"fmt"
+	"testing"
+)
+
+func TestPage(t *testing.T) {
+	PageRequest(context.Background(), "TestPageResult", 2, 10, PageRequestFunc{
+		RequestPage: func(ctx context.Context, pageSize int, page int) (interface{}, error) {
+			fmt.Printf("request -> page: %d, pagesize: %d\n", page, pageSize)
+			return nil, nil
+		},
+		PageResult: func(items interface{}) (total int, currentPageLen int, err error) {
+			return 8, 6, nil
+		},
+	})
+}

plugin/client/gitcontent.go

@@ -78,6 +78,7 @@ func (g *gitContent) Create(ctx context.Context, baseURL *duckv1.Addressable, pa
 	}
 	w.Close()
 	payload.Content = b.Bytes()
+
 	options = append(options, MetaOpts(g.meta), SecretOpts(g.secret), BodyOpts(payload.CreateRepoFileParams), ResultOpts(commitInfo))
 	if payload.Repository == "" {
 		return nil, errors.NewBadRequest("repo is empty string")