Changed logic search projects in gitlab adapter (goharbor#18529)

* fix(gitlab): change logic search projects usage search_namespaces

Signed-off-by: lxShaDoWxl <[email protected]>

* tests(gitlab): remove old data and actualization test

Signed-off-by: lxShaDoWxl <[email protected]>

* refactor(gitlab): added debug log

Signed-off-by: lxShaDoWxl <[email protected]>
(cherry picked from commit 7328062)

* lint(gitlab): fix import order

Signed-off-by: lxShaDoWxl <[email protected]>

---------

Signed-off-by: lxShaDoWxl <[email protected]>

src/pkg/reg/adapter/gitlab/adapter.go

@@ -15,6 +15,7 @@
 package gitlab
 
 import (
+	"net/url"
 	"strings"
 
 	"github.com/goharbor/harbor/src/lib/log"
@@ -127,6 +128,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 	} else {
 		pathPatterns = append(pathPatterns, nameFilter)
 	}
+	log.Debugf("Patterns: %v", pathPatterns)
 
 	for _, project := range projects {
 		if !project.RegistryEnabled {
@@ -143,8 +145,10 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 		}
 		for _, repository := range repositories {
 			if !existPatterns(repository.Path, pathPatterns) {
+				log.Debugf("Skipping repository path=%s and id=%d", repository.Path, repository.ID)
 				continue
 			}
+			log.Debugf("Search tags repository path=%s and id=%d", repository.Path, repository.ID)
 			vTags, err := a.clientGitlabAPI.getTags(project.ID, repository.ID)
 			if err != nil {
 				return nil, err
@@ -183,21 +187,12 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 
 func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) {
 	var projects []*Project
-	projectset := make(map[string]bool)
 	var err error
 	if len(pattern) > 0 {
 		names, ok := util.IsSpecificPath(pattern)
 		if ok {
 			for _, name := range names {
-				substrings := strings.Split(name, "/")
-				if len(substrings) < 2 {
-					continue
-				}
-				if _, ok := projectset[substrings[1]]; ok {
-					continue
-				}
-				projectset[substrings[1]] = true
-				var projectsByName, err = a.clientGitlabAPI.getProjectsByName(substrings[1])
+				var projectsByName, err = a.clientGitlabAPI.getProjectsByName(url.QueryEscape(name))
 				if err != nil {
 					return nil, err
 				}
@@ -207,20 +202,20 @@ func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) {
 				projects = append(projects, projectsByName...)
 			}
 		} else {
-			substrings := strings.Split(pattern, "/")
-			if len(substrings) < 2 {
-				return projects, nil
-			}
-			projectName := substrings[1]
-			if projectName == "*" {
-				return projects, nil
+			projectName := ""
+			for i, substring := range strings.Split(pattern, "/") {
+				if strings.Contains(substring, "*") {
+					if i != 0 {
+						break
+					}
+				} else {
+					projectName += substring + "/"
+				}
 			}
-			projectName = strings.Trim(projectName, "*")
-
-			if strings.Contains(projectName, "*") {
+			if projectName == "" {
 				return projects, nil
 			}
-			projects, err = a.clientGitlabAPI.getProjectsByName(projectName)
+			projects, err = a.clientGitlabAPI.getProjectsByName(url.QueryEscape(projectName))
 			if err != nil {
 				return nil, err
 			}

src/pkg/reg/adapter/gitlab/adapter_test.go

@@ -38,16 +38,10 @@ func getServer(t *testing.T) *httptest.Server {
 		w.Header().Set("X-Next-Page", "")
 
 		switch search {
-		case "dev-docker":
+		case "library/dev-docker", "library", "library/", "dev-docker/", "dev-docker":
 			mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json")
 			break
-		case "dev-":
-			mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json")
-			break
-		case "-docker":
-			mustWriteHTTPResponse(t, w, "testdata/projects/-docker.json")
-			break
-		case "":
+		case "", "library/dockers":
 			mustWriteHTTPResponse(t, w, "testdata/projects/all.json")
 			break
 		default:

src/pkg/reg/adapter/gitlab/client.go

@@ -25,6 +25,7 @@ import (
 
 	common_http "github.com/goharbor/harbor/src/common/http"
 	liberrors "github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/pkg/reg/model"
 	"github.com/goharbor/harbor/src/pkg/reg/util"
 )
@@ -82,7 +83,7 @@ func (c *Client) getProjects() ([]*Project, error) {
 
 func (c *Client) getProjectsByName(name string) ([]*Project, error) {
 	var projects []*Project
-	urlAPI := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=1&per_page=50", c.url, name)
+	urlAPI := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=true&search_namespaces=true&per_page=50", c.url, name)
 	if err := c.GetAndIteratePagination(urlAPI, &projects); err != nil {
 		return nil, err
 	}
@@ -94,6 +95,7 @@ func (c *Client) getRepositories(projectID int64) ([]*Repository, error) {
 	if err := c.GetAndIteratePagination(urlAPI, &repositories); err != nil {
 		return nil, err
 	}
+	log.Debugf("Count repositories %d in project %d", len(repositories), projectID)
 	return repositories, nil
 }
 
@@ -103,6 +105,7 @@ func (c *Client) getTags(projectID int64, repositoryID int64) ([]*Tag, error) {
 	if err := c.GetAndIteratePagination(urlAPI, &tags); err != nil {
 		return nil, err
 	}
+	log.Debugf("Count tags %d in repository %d, and project  %d", len(tags), repositoryID, projectID)
 	return tags, nil
 }
 
@@ -113,7 +116,6 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
 	if err != nil {
 		return err
 	}
-
 	rv := reflect.ValueOf(v)
 	if rv.Kind() != reflect.Ptr {
 		return errors.New("v should be a pointer to a slice")
@@ -122,7 +124,7 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
 	if elemType.Kind() != reflect.Slice {
 		return errors.New("v should be a pointer to a slice")
 	}
-
+	log.Debugf("Gitlab request %s", urlAPI)
 	resources := reflect.Indirect(reflect.New(elemType))
 	for len(endpoint) > 0 {
 		req, err := c.newRequest(http.MethodGet, endpoint, nil)