From 3798e5de552a86119c219b0d485d542007795913 Mon Sep 17 00:00:00 2001 From: Jack Vine Date: Thu, 7 Jul 2022 17:13:57 +0930 Subject: [PATCH 1/6] Implemented basic search API endpoint, just enough to work with Unity --- modules/packages/npm/creator.go | 10 ++++++++++ routers/api/packages/api.go | 3 +++ routers/api/packages/npm/api.go | 21 +++++++++++++++++++++ routers/api/packages/npm/npm.go | 21 +++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 88ce55ecdbef8..25333cfba3ee1 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -96,6 +96,16 @@ type PackageDistribution struct { NpmSignature string `json:"npm-signature,omitempty"` } +type PackagesSearch struct { + Objects []*PackagesSearchObject `json:"objects"` + Total int `json:"total"` + Time map[string]time.Time `json:"time,omitempty"` +} + +type PackagesSearchObject struct { + Package *PackageMetadataVersion `json:"package"` +} + // User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package type User struct { Username string `json:"username,omitempty"` diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index b5fdc739d7c10..999c4c2d60178 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -215,6 +215,9 @@ func Routes() *web.Route { r.Delete("", npm.DeletePackageTag) }, reqPackageAccess(perm.AccessModeWrite)) }) + r.Group("/-/v1/search", func() { + r.Get("", npm.PackagesSearch) + }) }) r.Group("/pypi", func() { r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index 56c897704398d..5c23ee9ab9838 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -71,3 +71,24 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package }, } } + +func createPackagesSearchResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackagesSearch { + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) + + versions := make([]*npm_module.PackagesSearchObject, len(pds)) + for i := range pds { + versions[i] = &npm_module.PackagesSearchObject{ + Package: createPackageMetadataVersion(registryURL, pds[i]), + } + } + + // TODO: Only show latest versions + // latest := pds[len(pds)-1] + + return &npm_module.PackagesSearch{ + Objects: versions, + Total: len(versions), + } +} diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index d127134d44558..b51a012e0d3c3 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -291,3 +291,24 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo return committer.Commit() } + +func PackagesSearch(ctx *context.Context) { + pvs, err := packages_model.GetVersionsByPackageType(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createPackagesSearchResponse( + setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/npm", + pds, + ) + + ctx.JSON(http.StatusOK, resp) +} From ba326d07ed7ceae53317bd1bbba14c7ee6826531 Mon Sep 17 00:00:00 2001 From: Jack Vine Date: Thu, 7 Jul 2022 10:34:26 +0000 Subject: [PATCH 2/6] Search query now working, only shows the latest version --- routers/api/packages/npm/api.go | 3 --- routers/api/packages/npm/npm.go | 9 ++++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index 5c23ee9ab9838..a70c257f1d765 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -84,9 +84,6 @@ func createPackagesSearchResponse(registryURL string, pds []*packages_model.Pack } } - // TODO: Only show latest versions - // latest := pds[len(pds)-1] - return &npm_module.PackagesSearch{ Objects: versions, Total: len(versions), diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index b51a012e0d3c3..60250d9c6bcff 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -293,7 +293,14 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo } func PackagesSearch(ctx *context.Context) { - pvs, err := packages_model.GetVersionsByPackageType(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm) + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeNpm, + Name: packages_model.SearchValue{ + ExactMatch: false, + Value: ctx.Req.URL.Query().Get("text"), + }, + }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return From 6af8e0ea2785d026de15b1e433b0f7053f2a1fdd Mon Sep 17 00:00:00 2001 From: Jack Vine Date: Fri, 8 Jul 2022 09:28:23 +0930 Subject: [PATCH 3/6] Removed superfluous sorting of package descriptors --- routers/api/packages/npm/api.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index a70c257f1d765..ee917609fd0d2 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -73,10 +73,6 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package } func createPackagesSearchResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackagesSearch { - sort.Slice(pds, func(i, j int) bool { - return pds[i].SemVer.LessThan(pds[j].SemVer) - }) - versions := make([]*npm_module.PackagesSearchObject, len(pds)) for i := range pds { versions[i] = &npm_module.PackagesSearchObject{ From 97b9f1b853fa8f98f01ab120e2da712c93ef62da Mon Sep 17 00:00:00 2001 From: Jack Vine Date: Fri, 8 Jul 2022 09:33:44 +0930 Subject: [PATCH 4/6] Renamed all PackagesSearch to PackageSearch --- modules/packages/npm/creator.go | 6 +++--- routers/api/packages/api.go | 2 +- routers/api/packages/npm/api.go | 8 ++++---- routers/api/packages/npm/npm.go | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 25333cfba3ee1..92f1b62e3aa10 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -96,13 +96,13 @@ type PackageDistribution struct { NpmSignature string `json:"npm-signature,omitempty"` } -type PackagesSearch struct { - Objects []*PackagesSearchObject `json:"objects"` +type PackageSearch struct { + Objects []*PackageSearchObject `json:"objects"` Total int `json:"total"` Time map[string]time.Time `json:"time,omitempty"` } -type PackagesSearchObject struct { +type PackageSearchObject struct { Package *PackageMetadataVersion `json:"package"` } diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 999c4c2d60178..9f636ea8a59fc 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -216,7 +216,7 @@ func Routes() *web.Route { }, reqPackageAccess(perm.AccessModeWrite)) }) r.Group("/-/v1/search", func() { - r.Get("", npm.PackagesSearch) + r.Get("", npm.PackageSearch) }) }) r.Group("/pypi", func() { diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index ee917609fd0d2..41bff5ff97ca7 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -72,15 +72,15 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package } } -func createPackagesSearchResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackagesSearch { - versions := make([]*npm_module.PackagesSearchObject, len(pds)) +func createPackageSearchResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackageSearch { + versions := make([]*npm_module.PackageSearchObject, len(pds)) for i := range pds { - versions[i] = &npm_module.PackagesSearchObject{ + versions[i] = &npm_module.PackageSearchObject{ Package: createPackageMetadataVersion(registryURL, pds[i]), } } - return &npm_module.PackagesSearch{ + return &npm_module.PackageSearch{ Objects: versions, Total: len(versions), } diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 60250d9c6bcff..d7228c49f71b3 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -292,7 +292,7 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo return committer.Commit() } -func PackagesSearch(ctx *context.Context) { +func PackageSearch(ctx *context.Context) { pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: packages_model.TypeNpm, @@ -312,7 +312,7 @@ func PackagesSearch(ctx *context.Context) { return } - resp := createPackagesSearchResponse( + resp := createPackageSearchResponse( setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/npm", pds, ) From 7fb266b75ccd1b385ffe51bd3e744a4ad0155e67 Mon Sep 17 00:00:00 2001 From: Jack Vine Date: Fri, 8 Jul 2022 10:19:26 +0930 Subject: [PATCH 5/6] PackageSearch changed to better match NPM example --- modules/packages/npm/creator.go | 20 +++++++++++++++++--- routers/api/packages/npm/api.go | 17 ++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 92f1b62e3aa10..0d3e92ae8751d 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -98,12 +98,26 @@ type PackageDistribution struct { type PackageSearch struct { Objects []*PackageSearchObject `json:"objects"` - Total int `json:"total"` - Time map[string]time.Time `json:"time,omitempty"` + Total int `json:"total"` + Time map[string]time.Time `json:"time,omitempty"` } type PackageSearchObject struct { - Package *PackageMetadataVersion `json:"package"` + Package *PackageSearchPackage `json:"package"` +} + +type PackageSearchPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Links *PackageSearchPackageLinks `json:"links"` + Maintainers []User `json:"maintainers,omitempty"` +} + +type PackageSearchPackageLinks struct { + Registry string `json:"npm"` + Homepage string `json:"homepage,omitempty"` + Repository string `json:"repository,omitempty"` } // User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index 41bff5ff97ca7..01c370d83d7ec 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -75,8 +75,23 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package func createPackageSearchResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackageSearch { versions := make([]*npm_module.PackageSearchObject, len(pds)) for i := range pds { + pd := pds[i] + metadata := createPackageMetadataVersion(registryURL, pd) + + pkg := &npm_module.PackageSearchPackage{ + Name: pd.Package.Name, + Version: pd.Version.Version, + Description: metadata.Description, + Links: &npm_module.PackageSearchPackageLinks{ + Registry: registryURL, + Homepage: metadata.Homepage, + Repository: metadata.Repository.URL, + }, + Maintainers: metadata.Maintainers, + } + versions[i] = &npm_module.PackageSearchObject{ - Package: createPackageMetadataVersion(registryURL, pds[i]), + Package: pkg, } } From 363174d7c09ec8eef9dfacaa9be5bfacf637ca1a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 18 Sep 2022 11:31:47 +0000 Subject: [PATCH 6/6] Implement npm search api. --- docs/content/doc/packages/npm.en-us.md | 5 +++ modules/packages/npm/creator.go | 10 +++-- routers/api/packages/npm/api.go | 48 ++++++++++++---------- routers/api/packages/npm/npm.go | 10 +++-- tests/integration/api_packages_npm_test.go | 31 ++++++++++++++ 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 122f306ee5e3a..decb3f743c419 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -127,6 +127,10 @@ npm dist-tag add test_package@1.0.2 release The tag name must not be a valid version. All tag names which are parsable as a version are rejected. +## Search packages + +The registry supports [searching](https://docs.npmjs.com/cli/v7/commands/npm-search/) but does not support special search qualifiers like `author:gitea`. + ## Supported commands ``` @@ -136,4 +140,5 @@ npm publish npm unpublish npm dist-tag npm view +npm search ``` diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 0d3e92ae8751d..2ed4d262481d1 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -98,8 +98,7 @@ type PackageDistribution struct { type PackageSearch struct { Objects []*PackageSearchObject `json:"objects"` - Total int `json:"total"` - Time map[string]time.Time `json:"time,omitempty"` + Total int64 `json:"total"` } type PackageSearchObject struct { @@ -107,11 +106,16 @@ type PackageSearchObject struct { } type PackageSearchPackage struct { + Scope string `json:"scope"` Name string `json:"name"` Version string `json:"version"` + Date time.Time `json:"date"` Description string `json:"description"` + Author User `json:"author"` + Publisher User `json:"publisher"` + Maintainers []User `json:"maintainers"` + Keywords []string `json:"keywords,omitempty"` Links *PackageSearchPackageLinks `json:"links"` - Maintainers []User `json:"maintainers,omitempty"` } type PackageSearchPackageLinks struct { diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index 09d3e4d010607..d1027763a836a 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -75,31 +75,37 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package } } -func createPackageSearchResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackageSearch { - versions := make([]*npm_module.PackageSearchObject, len(pds)) - for i := range pds { - pd := pds[i] - metadata := createPackageMetadataVersion(registryURL, pd) - - pkg := &npm_module.PackageSearchPackage{ - Name: pd.Package.Name, - Version: pd.Version.Version, - Description: metadata.Description, - Links: &npm_module.PackageSearchPackageLinks{ - Registry: registryURL, - Homepage: metadata.Homepage, - Repository: metadata.Repository.URL, - }, - Maintainers: metadata.Maintainers, - } +func createPackageSearchResponse(pds []*packages_model.PackageDescriptor, total int64) *npm_module.PackageSearch { + objects := make([]*npm_module.PackageSearchObject, 0, len(pds)) + for _, pd := range pds { + metadata := pd.Metadata.(*npm_module.Metadata) - versions[i] = &npm_module.PackageSearchObject{ - Package: pkg, + scope := metadata.Scope + if scope == "" { + scope = "unscoped" } + + objects = append(objects, &npm_module.PackageSearchObject{ + Package: &npm_module.PackageSearchPackage{ + Scope: scope, + Name: metadata.Name, + Version: pd.Version.Version, + Date: pd.Version.CreatedUnix.AsLocalTime(), + Description: metadata.Description, + Author: npm_module.User{Name: metadata.Author}, + Publisher: npm_module.User{Name: pd.Owner.Name}, + Maintainers: []npm_module.User{}, // npm cli needs this field + Keywords: metadata.Keywords, + Links: &npm_module.PackageSearchPackageLinks{ + Registry: pd.FullWebLink(), + Homepage: metadata.ProjectURL, + }, + }, + }) } return &npm_module.PackageSearch{ - Objects: versions, - Total: len(versions), + Objects: objects, + Total: total, } } diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index fd773ce38c26b..2989ce6e7f6bd 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -352,13 +352,17 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo } func PackageSearch(ctx *context.Context) { - pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: packages_model.TypeNpm, Name: packages_model.SearchValue{ ExactMatch: false, - Value: ctx.Req.URL.Query().Get("text"), + Value: ctx.FormTrim("text"), }, + Paginator: db.NewAbsoluteListOptions( + ctx.FormInt("from"), + ctx.FormInt("size"), + ), }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -372,8 +376,8 @@ func PackageSearch(ctx *context.Context) { } resp := createPackageSearchResponse( - setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/npm", pds, + total, ) ctx.JSON(http.StatusOK, resp) diff --git a/tests/integration/api_packages_npm_test.go b/tests/integration/api_packages_npm_test.go index fe6cea1cb68da..14ed681be3a7a 100644 --- a/tests/integration/api_packages_npm_test.go +++ b/tests/integration/api_packages_npm_test.go @@ -224,6 +224,37 @@ func TestPackageNpm(t *testing.T) { test(t, http.StatusOK, packageTag2) }) + t.Run("Search", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf("/api/packages/%s/npm/-/v1/search", user.Name) + + cases := []struct { + Query string + Skip int + Take int + ExpectedTotal int64 + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 0, 10, 1, 1}, + {"gitea", 0, 10, 0, 0}, + {"test", 0, 10, 1, 1}, + {"test", 1, 10, 1, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s?text=%s&from=%d&size=%d", url, c.Query, c.Skip, c.Take)) + resp := MakeRequest(t, req, http.StatusOK) + + var result npm.PackageSearch + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Objects, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + t.Run("Delete", func(t *testing.T) { defer tests.PrintCurrentTest(t)()