From 57e97bde5f74567fd35623e047a9a5bffc7cf828 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 12 Jul 2022 17:51:02 -0400 Subject: [PATCH] `download-starter-projects` functions added to Registry Library APIs (#126) Signed-off-by: Michael Valdron --- registry-library/README.md | 42 ++ registry-library/cmd/root.go | 30 +- registry-library/go.mod | 2 +- registry-library/go.sum | 29 + registry-library/library/library.go | 346 ++++++--- registry-library/library/library_test.go | 682 +++++++++++++++++- registry-library/library/util.go | 145 ++++ registry-library/library/util_test.go | 117 +++ .../index/generator/schema/schema.go | 46 +- registry-library/vendor/modules.txt | 2 +- .../pkg/tests/devfilelibrary_tests.go | 33 +- 11 files changed, 1298 insertions(+), 176 deletions(-) create mode 100644 registry-library/library/util.go create mode 100644 registry-library/library/util_test.go diff --git a/registry-library/README.md b/registry-library/README.md index 28f8e34d0..2d1d88ebd 100644 --- a/registry-library/README.md +++ b/registry-library/README.md @@ -122,3 +122,45 @@ Supported devfile media types can be found in the latest version of [library.go] } ``` +#### Download the starter project + +1. Download starter project in-memory +```go +var bytes []byte +var err error + +... + +starterProject := "springbootproject" +bytes, err = registryLibrary.DownloadStarterProjectAsBytes(registryURL, + stack, starterProject, options) +if err != nil { + return err +} +``` +2. Download starter project archive to filesystem path +```go +starterProject := "springbootproject" +path := fmt.Sprintf("%s.zip", starterProject) +err := registryLibrary.DownloadStarterProject(path, registryURL, stack, starterProject, options) +if err != nil { + return err +} +``` + +```sh +ls # => devfile.yaml springbootproject.zip +``` +3. Download starter project archive and extract to filesystem path +```go +starterProject := "springbootproject" +path := "." +err := registryLibrary.DownloadStarterProjectAsDir(path, registryURL, stack, starterProject, options) +if err != nil { + return err +} +``` + +```sh +ls # => devfile.yaml pom.xml HELP.md mvnw mvnw.cmd src +``` diff --git a/registry-library/cmd/root.go b/registry-library/cmd/root.go index a78c821c3..7f1e2d462 100644 --- a/registry-library/cmd/root.go +++ b/registry-library/cmd/root.go @@ -18,6 +18,7 @@ package cmd import ( "fmt" "os" + "path/filepath" "github.com/spf13/cobra" @@ -147,7 +148,34 @@ func init() { listCmd.Flags().BoolVar(&skipTLSVerify, "skip-tls-verify", false, "skip TLS verification") listCmd.Flags().StringVar(&user, "user", "", "consumer name") - rootCmd.AddCommand(pullCmd, listCmd) + var downloadCmd = &cobra.Command{ + Use: "download ", + Short: "Downloads starter project", + Args: cobra.ExactArgs(3), + Run: func(cmd *cobra.Command, args []string) { + registry, stack, starterProject := args[0], args[1], args[2] + var err error + + options := library.RegistryOptions{ + NewIndexSchema: newIndexSchema, + Telemetry: library.TelemetryData{ + User: "user", + }, + SkipTLSVerify: skipTLSVerify, + } + + err = library.DownloadStarterProjectAsDir(filepath.Join(destDir, starterProject), registry, stack, starterProject, options) + if err != nil { + fmt.Printf("failed to download starter project %s: %v\n", starterProject, err) + } + }, + } + downloadCmd.Flags().StringVar(&destDir, "context", ".", "destination directory that stores stack resources") + downloadCmd.Flags().BoolVar(&skipTLSVerify, "skip-tls-verify", false, "skip TLS verification") + downloadCmd.Flags().BoolVar(&newIndexSchema, "new-index-schema", false, "use new index schema") + downloadCmd.Flags().StringVar(&user, "user", "", "consumer name") + + rootCmd.AddCommand(pullCmd, listCmd, downloadCmd) } // initConfig reads in config file and ENV variables if set. diff --git a/registry-library/go.mod b/registry-library/go.mod index 696f47247..2704b88cb 100644 --- a/registry-library/go.mod +++ b/registry-library/go.mod @@ -4,7 +4,7 @@ go 1.14 require ( github.com/containerd/containerd v1.5.9 - github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e + github.com/devfile/registry-support/index/generator v0.0.0-20220624203950-e7282a4695b6 github.com/hashicorp/go-version v1.4.0 github.com/mitchellh/go-homedir v1.1.0 github.com/spf13/cobra v1.2.1 diff --git a/registry-library/go.sum b/registry-library/go.sum index 32aad0883..06016f760 100644 --- a/registry-library/go.sum +++ b/registry-library/go.sum @@ -86,23 +86,27 @@ github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:m github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -281,9 +285,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/devfile/api/v2 v2.0.0-20211021164004-dabee4e633ed/go.mod h1:d99eTN6QxgzihOOFyOZA+VpUyD4Q1pYRYHZ/ci9J96Q= +github.com/devfile/api/v2 v2.0.0-20220117162434-6e6e6a8bc14c/go.mod h1:d99eTN6QxgzihOOFyOZA+VpUyD4Q1pYRYHZ/ci9J96Q= github.com/devfile/library v1.2.1-0.20211104222135-49d635cb492f/go.mod h1:uFZZdTuRqA68FVe/JoJHP92CgINyQkyWnM2Qyiim+50= +github.com/devfile/library v1.2.1-0.20220308191614-f0f7e11b17de/go.mod h1:GSPfJaBg0+bBjBHbwBE5aerJLH6tWGQu2q2rHYd9czM= github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e h1:3WAjUoyAmCBzUpx+sO0dSgkH74uSW1y886wGbojD2D8= github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e/go.mod h1:iRPBxs+ZjfLEduVXpCCIOzdD2588Zv9OCs/CcXMcCCY= +github.com/devfile/registry-support/index/generator v0.0.0-20220624203950-e7282a4695b6 h1:bTbZxKSjF9xfiUuOKpoyX7P/ZcnIRy993+JBvkQ91hw= +github.com/devfile/registry-support/index/generator v0.0.0-20220624203950-e7282a4695b6/go.mod h1:1fyDJL+fPHtcrYA6yjSVWeLmXmjCNth0d5Rq1rvtryc= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -322,6 +330,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -337,6 +346,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -346,6 +356,12 @@ github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXt github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -508,6 +524,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4= github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -531,7 +548,9 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -549,6 +568,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -583,6 +603,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -756,6 +777,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -835,6 +857,7 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= @@ -883,6 +906,7 @@ golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -895,6 +919,7 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -983,6 +1008,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1098,6 +1124,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1323,6 +1350,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1338,6 +1366,7 @@ gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/registry-library/library/library.go b/registry-library/library/library.go index 1abfeebf8..a0b360a9d 100644 --- a/registry-library/library/library.go +++ b/registry-library/library/library.go @@ -16,14 +16,12 @@ limitations under the License. package library import ( - "archive/tar" - "compress/gzip" + "archive/zip" "crypto/tls" "encoding/json" "fmt" "io" "io/ioutil" - "log" "net/http" "net/url" "os" @@ -247,64 +245,12 @@ func PrintRegistry(registryURLs string, devfileType string, options RegistryOpti // PullStackByMediaTypesFromRegistry pulls a specified stack with allowed media types from a given registry URL to the destination directory. // OWNERS files present in the registry will be excluded func PullStackByMediaTypesFromRegistry(registry string, stack string, allowedMediaTypes []string, destDir string, options RegistryOptions) error { - var requestVersion string - if strings.Contains(stack, ":") { - stackWithVersion := strings.Split(stack, ":") - stack = stackWithVersion[0] - requestVersion = stackWithVersion[1] - } - // Get the registry index - registryIndex, err := GetRegistryIndex(registry, options, indexSchema.StackDevfileType) + // Get stack link + stackLink, err := GetStackLink(registry, stack, options) if err != nil { return err } - // Parse the index to get the specified stack's metadata in the index - var stackIndex indexSchema.Schema - exist := false - for _, item := range registryIndex { - if item.Name == stack { - stackIndex = item - exist = true - break - } - } - if !exist { - return fmt.Errorf("stack %s does not exist in the registry %s", stack, registry) - } - var stackLink string - - if options.NewIndexSchema { - latestVersionIndex := 0 - latest, err := versionpkg.NewVersion(stackIndex.Versions[latestVersionIndex].Version) - if err != nil { - return fmt.Errorf("failed to parse the stack version %s for stack %s", stackIndex.Versions[latestVersionIndex].Version, stack) - } - for index, version := range stackIndex.Versions { - if (requestVersion == "" && version.Default) || (version.Version == requestVersion) { - stackLink = version.Links["self"] - break - } else if requestVersion == "latest" { - current, err := versionpkg.NewVersion(version.Version) - if err != nil { - return fmt.Errorf("failed to parse the stack version %s for stack %s", version.Version, stack) - } - if current.GreaterThan(latest) { - latestVersionIndex = index - latest = current - } - } - } - if requestVersion == "latest" { - stackLink = stackIndex.Versions[latestVersionIndex].Links["self"] - } - if stackLink == "" { - return fmt.Errorf("the requested verion %s for stack %s does not exist in the registry %s", requestVersion, stack, registry) - } - } else { - stackLink = stackIndex.Links["self"] - } - // Pull stack initialization ctx := orasctx.Background() urlObj, err := url.Parse(registry) @@ -356,77 +302,263 @@ func PullStackFromRegistry(registry string, stack string, destDir string, option return PullStackByMediaTypesFromRegistry(registry, stack, DevfileAllMediaTypesList, destDir, options) } -// decompress extracts the archive file -func decompress(targetDir string, tarFile string, excludeFiles []string) error { - reader, err := os.Open(tarFile) - if err != nil { +// DownloadStarterProjectAsDir downloads a specified starter project archive and extracts it to given path +func DownloadStarterProjectAsDir(path string, registryURL string, stack string, starterProject string, options RegistryOptions) error { + var err error + + // Create temp path to download archive + archivePath := filepath.Join(os.TempDir(), fmt.Sprintf("%s.zip", starterProject)) + + // Download starter project archive to temp path + if err = DownloadStarterProject(archivePath, registryURL, stack, starterProject, options); err != nil { return err } - defer reader.Close() - gzReader, err := gzip.NewReader(reader) + // Open archive reader + archive, err := zip.OpenReader(archivePath) if err != nil { - return err + return fmt.Errorf("error opening downloaded starter project archive: %v", err) } - defer gzReader.Close() + defer archive.Close() - tarReader := tar.NewReader(gzReader) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } else if err != nil { - return err + // Extract files from starter project archive to specified directory path + for _, file := range archive.File { + filePath := filepath.Join(path, file.Name) + + // validate extracted filepath + if !strings.HasPrefix(filePath, filepath.Clean(path)+string(os.PathSeparator)) { + return fmt.Errorf("invalid file path %s", filePath) } - if isExcluded(header.Name, excludeFiles) { + + // if file is a directory, create it in destination and continue to next file + if file.FileInfo().IsDir() { + os.MkdirAll(filePath, os.ModePerm) continue } - target := path.Join(targetDir, header.Name) - switch header.Typeflag { - case tar.TypeDir: - err = os.MkdirAll(target, os.FileMode(header.Mode)) - if err != nil { - return err - } - case tar.TypeReg: - w, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) - if err != nil { - return err - } - _, err = io.Copy(w, tarReader) - if err != nil { - return err - } - w.Close() - default: - log.Printf("Unsupported type: %v", header.Typeflag) + // ensure parent directory of current file is created in destination + if err = os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + return fmt.Errorf("error creating parent directory %s: %v", filepath.Dir(filePath), err) + } + + // open destination file + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) + if err != nil { + return fmt.Errorf("error opening destination file at %s: %v", filePath, err) + } + + // open source file in archive + srcFile, err := file.Open() + if err != nil { + return fmt.Errorf("error opening source file %s in archive %s: %v", file.Name, archivePath, err) + } + + // extract source file to destination file + if _, err = io.Copy(dstFile, srcFile); err != nil { + return fmt.Errorf("error extracting file %s from archive %s to destination at %s: %v", file.Name, archivePath, filePath, err) } + + dstFile.Close() + srcFile.Close() } return nil } -func isExcluded(name string, excludeFiles []string) bool { - basename := filepath.Base(name) - for _, excludeFile := range excludeFiles { - if basename == excludeFile { - return true +// DownloadStarterProject downloads a specified starter project archive to a given path +func DownloadStarterProject(path string, registryURL string, stack string, starterProject string, options RegistryOptions) error { + var fileStream *os.File + + // Download Starter Project archive bytes + bytes, err := DownloadStarterProjectAsBytes(registryURL, stack, starterProject, options) + if err != nil { + return err + } + + // Error if parent directory does not exist + if _, err = os.Stat(filepath.Dir(path)); os.IsNotExist(err) { + return fmt.Errorf("parent directory '%s' does not exist: %v", filepath.Dir(path), err) + } + + // If file does not exist, create a new one + // Else open existing for overwriting + if _, err = os.Stat(path); os.IsNotExist(err) { + fileStream, err = os.Create(path) + if err != nil { + return fmt.Errorf("failed to create file '%s': %v", path, err) + } + } else { + fileStream, err = os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to open file '%s': %v", path, err) + } + } + defer fileStream.Close() + + // Write downloaded bytes to file + _, err = fileStream.Write(bytes) + if err != nil { + return fmt.Errorf("failed writing to '%s': %v", path, err) + } + + return nil +} + +// DownloadStarterProjectAsBytes downloads the file bytes of a specified starter project archive and return these bytes +func DownloadStarterProjectAsBytes(registryURL string, stack string, starterProject string, options RegistryOptions) ([]byte, error) { + stackName, _, err := SplitVersionFromStack(stack) + if err != nil { + return nil, fmt.Errorf("problem in stack/version tag: %v", err) + } + + exists, err := IsStarterProjectExists(registryURL, stack, starterProject, options) + if err != nil { + return nil, err + } else if !exists { + return nil, fmt.Errorf("the starter project '%s' does not exist under the stack '%s'", starterProject, stackName) + } + + urlObj, err := url.Parse(registryURL) + if err != nil { + return nil, err + } + + url := fmt.Sprintf("%s/%s", urlObj.String(), path.Join("devfiles", stackName, "starter-projects", starterProject)) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + setHeaders(&req.Header, options) + + httpClient := &http.Client{ + Transport: &http.Transport{ + ResponseHeaderTimeout: responseHeaderTimeout, + TLSClientConfig: &tls.Config{InsecureSkipVerify: options.SkipTLSVerify}, + }, + Timeout: httpRequestTimeout, + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + + // Return downloaded starter project as bytes or error if unsuccessful. + return ioutil.ReadAll(resp.Body) +} + +// IsStarterProjectExists checks if starter project exists for a given stack +func IsStarterProjectExists(registryURL string, stack string, starterProject string, options RegistryOptions) (bool, error) { + // Get stack index + stackIndex, err := GetStackIndex(registryURL, stack, options) + if err != nil { + return false, err + } + + // Check if starter project exists in the stack index + exists := false + for _, sp := range stackIndex.StarterProjects { + if sp == starterProject { + exists = true + break + } + } + + if !exists && options.NewIndexSchema { + var starterProjects []string + + for _, version := range stackIndex.Versions { + starterProjects = append(starterProjects, version.StarterProjects...) + } + + exists = false + for _, sp := range starterProjects { + if sp == starterProject { + exists = true + break + } + } + + return exists, nil + } else { + return exists, nil + } +} + +// GetStackLink returns the slug needed to pull a specified stack from a registry URL +func GetStackLink(registryURL string, stack string, options RegistryOptions) (string, error) { + var stackLink string + + // Get stack index + stackIndex, err := GetStackIndex(registryURL, stack, options) + if err != nil { + return "", err + } + + // Split version from stack label if specified + stack, requestVersion, err := SplitVersionFromStack(stack) + if err != nil { + return "", fmt.Errorf("problem in stack/version tag: %v", err) + } + + if options.NewIndexSchema { + latestVersionIndex := 0 + latest, err := versionpkg.NewVersion(stackIndex.Versions[latestVersionIndex].Version) + if err != nil { + return "", fmt.Errorf("failed to parse the stack version %s for stack %s", stackIndex.Versions[latestVersionIndex].Version, stack) } + for index, version := range stackIndex.Versions { + if (requestVersion == "" && version.Default) || (version.Version == requestVersion) { + stackLink = version.Links["self"] + break + } else if requestVersion == "latest" { + current, err := versionpkg.NewVersion(version.Version) + if err != nil { + return "", fmt.Errorf("failed to parse the stack version %s for stack %s", version.Version, stack) + } + if current.GreaterThan(latest) { + latestVersionIndex = index + latest = current + } + } + } + if requestVersion == "latest" { + stackLink = stackIndex.Versions[latestVersionIndex].Links["self"] + } + if requestVersion == "" && stackLink == "" { + return "", fmt.Errorf("no version specified for stack %s which no default version exists in the registry %s", stack, registryURL) + } else if stackLink == "" { + return "", fmt.Errorf("the requested version %s for stack %s does not exist in the registry %s", requestVersion, stack, registryURL) + } + } else { + stackLink = stackIndex.Links["self"] } - return false + + return stackLink, nil } -//setHeaders sets the request headers -func setHeaders(headers *http.Header, options RegistryOptions) { - t := options.Telemetry - if t.User != "" { - headers.Add("User", t.User) +// GetStackIndex returns the schema index of a specified stack +func GetStackIndex(registryURL string, stack string, options RegistryOptions) (indexSchema.Schema, error) { + // Get the registry index + registryIndex, err := GetRegistryIndex(registryURL, options, indexSchema.StackDevfileType) + if err != nil { + return indexSchema.Schema{}, err } - if t.Client != "" { - headers.Add("Client", t.Client) + + // Prune version from stack label if specified + stack, _, err = SplitVersionFromStack(stack) + if err != nil { + return indexSchema.Schema{}, fmt.Errorf("problem in stack/version tag: %v", err) } - if t.Locale != "" { - headers.Add("Locale", t.Locale) + + // Parse the index to get the specified stack's metadata in the index + for _, item := range registryIndex { + // Return index of specified stack if found + if item.Name == stack { + return item, nil + } } + + // Return error if stack index is not found + return indexSchema.Schema{}, fmt.Errorf("stack %s does not exist in the registry %s", stack, registryURL) } diff --git a/registry-library/library/library_test.go b/registry-library/library/library_test.go index ebc566820..e54b42729 100644 --- a/registry-library/library/library_test.go +++ b/registry-library/library/library_test.go @@ -1,20 +1,31 @@ package library import ( + "archive/zip" + bytespkg "bytes" "encoding/json" + "fmt" + "io" "net" "net/http" "net/http/httptest" + "net/url" + "os" + "path/filepath" "reflect" + "regexp" "strings" "testing" indexSchema "github.com/devfile/registry-support/index/generator/schema" ) -func TestGetRegistryIndex(t *testing.T) { - const serverIP = "127.0.0.1:8080" - archFilteredIndex := []indexSchema.Schema{ +const ( + serverIP = "127.0.0.1:8080" +) + +var ( + archFilteredIndex = []indexSchema.Schema{ { Name: "archindex1", Architectures: []string{"amd64, arm64"}, @@ -24,7 +35,7 @@ func TestGetRegistryIndex(t *testing.T) { }, } - schemaVersionFilteredIndex := []indexSchema.Schema{ + schemaVersionFilteredIndex = []indexSchema.Schema{ { Name: "indexSchema2.1", Versions: []indexSchema.Version{ @@ -45,7 +56,7 @@ func TestGetRegistryIndex(t *testing.T) { }, } - sampleFilteredIndex := []indexSchema.Schema{ + sampleFilteredIndex = []indexSchema.Schema{ { Name: "sampleindex1", }, @@ -54,7 +65,7 @@ func TestGetRegistryIndex(t *testing.T) { }, } - sampleFilteredV2Index := []indexSchema.Schema{ + sampleFilteredV2Index = []indexSchema.Schema{ { Name: "samplev2index1", }, @@ -63,25 +74,88 @@ func TestGetRegistryIndex(t *testing.T) { }, } - stackFilteredIndex := []indexSchema.Schema{ + stackFilteredIndex = []indexSchema.Schema{ { Name: "stackindex1", + Links: map[string]string{ + "self": "devfile-catalog/stackindex1:1.0.0", + }, + StarterProjects: []string{ + "stackindex1-starter", + }, }, { Name: "stackindex2", + Links: map[string]string{ + "self": "devfile-catalog/stackindex2:1.0.0", + }, + StarterProjects: []string{ + "stackindex2-starter", + }, }, } - stackFilteredV2Index := []indexSchema.Schema{ + stackFilteredV2Index = []indexSchema.Schema{ { Name: "stackv2index1", + Versions: []indexSchema.Version{{ + Version: "2.0.0", + Links: map[string]string{ + "self": "devfile-catalog/stackv2index1:2.0.0", + }, + StarterProjects: []string{ + "stackv2index1-starter", + }, + }, { + Version: "2.1.0", + Default: true, + Links: map[string]string{ + "self": "devfile-catalog/stackv2index1:2.1.0", + }, + StarterProjects: []string{ + "stackv2index1-starter", + }, + }, { + Version: "2.2.0", + Links: map[string]string{ + "self": "devfile-catalog/stackv2index1:2.2.0", + }, + StarterProjects: []string{ + "index1-starter", + }, + }}, }, { Name: "stackv2index2", + Versions: []indexSchema.Version{{ + Version: "2.0.0", + Links: map[string]string{ + "self": "devfile-catalog/stackv2index2:2.0.0", + }, + StarterProjects: []string{ + "stackv2index2-starter", + }, + }, { + Version: "2.1.0", + Links: map[string]string{ + "self": "devfile-catalog/stackv2index2:2.1.0", + }, + StarterProjects: []string{ + "stackv2index2-starter", + }, + }, { + Version: "2.2.0", + Links: map[string]string{ + "self": "devfile-catalog/stackv2index2:2.2.0", + }, + StarterProjects: []string{ + "index2-starter", + }, + }}, }, } - notFilteredIndex := []indexSchema.Schema{ + notFilteredIndex = []indexSchema.Schema{ { Name: "index1", }, @@ -90,7 +164,7 @@ func TestGetRegistryIndex(t *testing.T) { }, } - notFilteredV2Index := []indexSchema.Schema{ + notFilteredV2Index = []indexSchema.Schema{ { Name: "v2index1", }, @@ -98,34 +172,66 @@ func TestGetRegistryIndex(t *testing.T) { Name: "v2index2", }, } +) + +func setUpIndexHandle(indexUrl *url.URL) []indexSchema.Schema { + var data []indexSchema.Schema + + if strings.Contains(indexUrl.String(), "arch=amd64&arch=arm64") { + data = archFilteredIndex + } else if strings.Contains(indexUrl.String(), "maxSchemaVersion=2.2") && strings.Contains(indexUrl.String(), "minSchemaVersion=2.1") { + data = schemaVersionFilteredIndex + } else if indexUrl.Path == "/index/sample" { + data = sampleFilteredIndex + } else if indexUrl.Path == "/v2index/sample" { + data = sampleFilteredV2Index + } else if indexUrl.Path == "/index/stack" || indexUrl.Path == "/index" { + data = stackFilteredIndex + } else if indexUrl.Path == "/v2index/stack" || indexUrl.Path == "/v2index" { + data = stackFilteredV2Index + } else if indexUrl.Path == "/index/all" { + data = notFilteredIndex + } else if indexUrl.Path == "/v2index/all" { + data = notFilteredV2Index + } + + return data +} +func setUpTestServer(t *testing.T) (func(), error) { // Mocking the registry REST endpoints on a very basic level testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - var data []indexSchema.Schema + var bytes []byte var err error - if strings.Contains(r.URL.String(), "arch=amd64&arch=arm64") { - data = archFilteredIndex - } else if strings.Contains(r.URL.String(), "maxSchemaVersion=2.2") && strings.Contains(r.URL.String(), "minSchemaVersion=2.1") { - data = schemaVersionFilteredIndex - } else if r.URL.Path == "/index/sample" { - data = sampleFilteredIndex - } else if r.URL.Path == "/v2index/sample" { - data = sampleFilteredV2Index - } else if r.URL.Path == "/index/stack" || r.URL.Path == "/index" { - data = stackFilteredIndex - } else if r.URL.Path == "/v2index/stack" || r.URL.Path == "/v2index" { - data = stackFilteredV2Index - } else if r.URL.Path == "/index/all" { - data = notFilteredIndex - } else if r.URL.Path == "/v2index/all" { - data = notFilteredV2Index - } + if matched, err := regexp.MatchString(`/devfiles/[^/]+/starter-projects/[^/]+`, r.URL.Path); matched { + if err != nil { + t.Errorf("Unexpected error while matching url: %v", err) + return + } - bytes, err := json.MarshalIndent(&data, "", " ") - if err != nil { - t.Errorf("Unexpected error while doing json marshal: %v", err) + buffer := bytespkg.Buffer{} + writer := zip.NewWriter(&buffer) + + _, err = writer.Create("README.md") + if err != nil { + t.Errorf("error in creating testing starter project archive: %v", err) + return + } + + writer.Close() + + bytes = buffer.Bytes() + } else if strings.HasPrefix(r.URL.Path, "/index") || strings.HasPrefix(r.URL.Path, "/v2index") { + data := setUpIndexHandle(r.URL) + + bytes, err = json.MarshalIndent(&data, "", " ") + if err != nil { + t.Errorf("Unexpected error while doing json marshal: %v", err) + return + } + } else { + t.Errorf("Route %s was not found", r.URL.Path) return } @@ -137,8 +243,7 @@ func TestGetRegistryIndex(t *testing.T) { // create a listener with the desired port. l, err := net.Listen("tcp", serverIP) if err != nil { - t.Errorf("Unexpected error while creating listener: %v", err) - return + return testServer.Close, fmt.Errorf("Unexpected error while creating listener: %v", err) } // NewUnstartedServer creates a listener. Close that listener and replace @@ -147,7 +252,17 @@ func TestGetRegistryIndex(t *testing.T) { testServer.Listener = l testServer.Start() - defer testServer.Close() + + return testServer.Close, nil +} + +func TestGetRegistryIndex(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() tests := []struct { name string @@ -168,7 +283,7 @@ func TestGetRegistryIndex(t *testing.T) { }, }, devfileTypes: []indexSchema.DevfileType{indexSchema.StackDevfileType}, - wantSchemas: schemaVersionFilteredIndex, + wantSchemas: schemaVersionFilteredIndex, }, { name: "Get Arch Filtered Index", @@ -239,10 +354,503 @@ func TestGetRegistryIndex(t *testing.T) { if !test.wantErr && err != nil { t.Errorf("Unexpected err: %+v", err) } else if test.wantErr && err == nil { - t.Errorf("Expected error but got nil") + t.Error("Expected error but got nil") } else if !reflect.DeepEqual(gotSchemas, test.wantSchemas) { t.Errorf("Expected: %+v, \nGot: %+v", test.wantSchemas, gotSchemas) } }) } } + +func TestGetStackIndex(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() + + tests := []struct { + name string + url string + stack string + options RegistryOptions + wantSchema indexSchema.Schema + wantErr bool + }{ + { + name: "Get Stack Schema Index", + url: "http://" + serverIP, + stack: "stackindex1", + wantSchema: stackFilteredIndex[0], + }, + { + name: "Get V2 Stack Schema Index", + url: "http://" + serverIP, + stack: "stackv2index2", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantSchema: stackFilteredV2Index[1], + }, + { + name: "Get Non-Existent Stack Schema Index", + url: "http://" + serverIP, + stack: "fakestackindex", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotSchema, err := GetStackIndex(test.url, test.stack, test.options) + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if !reflect.DeepEqual(gotSchema, test.wantSchema) { + t.Errorf("Expected: %+v, \nGot: %+v", test.wantSchema, gotSchema) + } + }) + } +} + +func TestGetStackLink(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() + + tests := []struct { + name string + url string + stack string + options RegistryOptions + wantLink string + wantErr bool + }{ + { + name: "Get Stack Link", + url: "http://" + serverIP, + stack: "stackindex1", + wantLink: stackFilteredIndex[0].Links["self"], + }, + { + name: "Get V2 Stack default Link", + url: "http://" + serverIP, + stack: "stackv2index1", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantLink: stackFilteredV2Index[0].Versions[1].Links["self"], + }, + { + name: "Get V2 Stack latest Link", + url: "http://" + serverIP, + stack: "stackv2index2:latest", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantLink: stackFilteredV2Index[1].Versions[2].Links["self"], + }, + { + name: "Get V2 Stack Non-Existent Tagged Link", + url: "http://" + serverIP, + stack: "stackv2index2:faketag", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantErr: true, + }, + { + name: "Get V2 Stack Link with no default version", + url: "http://" + serverIP, + stack: "stackv2index2", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotLink, err := GetStackLink(test.url, test.stack, test.options) + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if !reflect.DeepEqual(gotLink, test.wantLink) { + t.Errorf("Expected: %+v, \nGot: %+v", test.wantLink, gotLink) + } + }) + } +} + +func TestIsStarterProjectExists(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() + + tests := []struct { + name string + url string + stack string + starterProject string + options RegistryOptions + wantExist bool + wantErr bool + }{ + { + name: "Starter Project Exists", + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "stackindex1-starter", + wantExist: true, + }, + { + name: "Starter Project Exists V2", + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "stackv2index1-starter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantExist: true, + }, + { + name: "Starter Project Does Not Exists", + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "fake-starter", + wantExist: false, + }, + { + name: "Starter Project Does Not Exists V2", + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "fake-starter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantExist: false, + }, + { + name: "Stack Does Not Exists", + url: "http://" + serverIP, + stack: "fake-stack", + starterProject: "fake-starter", + wantExist: false, + wantErr: true, + }, + { + name: "Stack Does Not Exists V2", + url: "http://" + serverIP, + stack: "fake-stack", + starterProject: "fake-starter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantExist: false, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + exists, err := IsStarterProjectExists(test.url, test.stack, test.starterProject, test.options) + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if !reflect.DeepEqual(exists, test.wantExist) { + t.Errorf("Expected: %+v, \nGot: %+v", test.wantExist, exists) + } + }) + } +} + +func TestDownloadStarterProjectAsBytes(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() + + tests := []struct { + name string + url string + stack string + starterProject string + options RegistryOptions + wantType string + wantErr bool + }{ + { + name: "Download Starter Project", + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "stackindex1-starter", + wantType: "application/zip", + }, + { + name: "Download Starter Project V2", + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "stackv2index1-starter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantType: "application/zip", + }, + { + name: "Download Starter Project from Fake Stack", + url: "http://" + serverIP, + stack: "fakestack", + starterProject: "fakestarter", + wantErr: true, + }, + { + name: "Download Fake Starter Project", + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "fakestarter", + wantErr: true, + }, + { + name: "Download Fake Starter Project V2", + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "fakestarter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotBytes, err := DownloadStarterProjectAsBytes(test.url, test.stack, test.starterProject, test.options) + + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if (test.wantErr || gotBytes == nil) && err == nil { + t.Error("Expected error but got nil") + } else if test.wantType != "" { + gotType := http.DetectContentType(gotBytes) + if !reflect.DeepEqual(gotType, test.wantType) { + t.Errorf("Expected: %+v, \nGot: %+v", test.wantType, gotType) + } + } + }) + } +} + +func TestDownloadStarterProject(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() + + tests := []struct { + name string + path string + url string + stack string + starterProject string + options RegistryOptions + wantType string + wantErr bool + }{ + { + name: "Download Starter Project", + path: filepath.Join(os.TempDir(), "test.zip"), + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "stackindex1-starter", + wantType: "application/zip", + }, + { + name: "Download Starter Project V2", + path: filepath.Join(os.TempDir(), "test.zip"), + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "stackv2index1-starter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantType: "application/zip", + }, + { + name: "Download Starter Project from Fake Stack", + path: filepath.Join(os.TempDir(), "test.zip"), + url: "http://" + serverIP, + stack: "fakestack", + starterProject: "fakestarter", + wantErr: true, + }, + { + name: "Download Fake Starter Project", + path: filepath.Join(os.TempDir(), "test.zip"), + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "fakestarter", + wantErr: true, + }, + { + name: "Download Fake Starter Project V2", + path: filepath.Join(os.TempDir(), "test.zip"), + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "fakestarter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantErr: true, + }, + { + name: "Download Starter Project to non-existent parent path", + path: filepath.Join(os.TempDir(), "dummy", "path", "to", "file.zip"), + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "stackindex1-starter", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := DownloadStarterProject(test.path, test.url, test.stack, test.starterProject, test.options) + + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if test.wantType != "" { + file, err := os.Open(test.path) + if err != nil { + t.Errorf("Unexpected err: %+v", err) + } + defer func() { + file.Close() + err := os.Remove(test.path) + if err != nil { + t.Errorf("Unexpected err: %+v", err) + } + }() + + gotBytes, err := io.ReadAll(file) + if err != nil { + t.Errorf("Unexpected err: %+v", err) + } + + gotType := http.DetectContentType(gotBytes) + if !reflect.DeepEqual(gotType, test.wantType) { + t.Errorf("Expected: %+v, \nGot: %+v", test.wantType, gotType) + } + } + }) + } +} + +func TestDownloadStarterProjectAsDir(t *testing.T) { + close, err := setUpTestServer(t) + if err != nil { + t.Error(err) + return + } + defer close() + + tests := []struct { + name string + path string + url string + stack string + starterProject string + options RegistryOptions + wantErr bool + }{ + { + name: "Download Starter Project", + path: filepath.Join(os.TempDir(), "stackindex1-starter"), + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "stackindex1-starter", + }, + { + name: "Download Starter Project V2", + path: filepath.Join(os.TempDir(), "stackv2index1-starter"), + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "stackv2index1-starter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + }, + { + name: "Download Starter Project from Fake Stack", + path: filepath.Join(os.TempDir(), "fakestarter"), + url: "http://" + serverIP, + stack: "fakestack", + starterProject: "fakestarter", + wantErr: true, + }, + { + name: "Download Fake Starter Project", + path: filepath.Join(os.TempDir(), "fakestarter"), + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "fakestarter", + wantErr: true, + }, + { + name: "Download Fake Starter Project V2", + path: filepath.Join(os.TempDir(), "fakestarter"), + url: "http://" + serverIP, + stack: "stackv2index1", + starterProject: "fakestarter", + options: RegistryOptions{ + NewIndexSchema: true, + }, + wantErr: true, + }, + { + name: "Download Starter Project to non-existent parent path", + path: filepath.Join(os.TempDir(), "stackindex1", "stackindex1-starter"), + url: "http://" + serverIP, + stack: "stackindex1", + starterProject: "stackindex1-starter", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := DownloadStarterProjectAsDir(test.path, test.url, test.stack, test.starterProject, test.options) + + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if err == nil { + fileinfo, err := os.Stat(test.path) + + if err != nil && os.IsNotExist(err) { + t.Errorf("Expected %s directory to exist.", test.path) + } else if err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if !fileinfo.IsDir() { + t.Errorf("%s was expected to be a directory but is a file.", test.path) + } + } + }) + } +} diff --git a/registry-library/library/util.go b/registry-library/library/util.go new file mode 100644 index 000000000..4a243d852 --- /dev/null +++ b/registry-library/library/util.go @@ -0,0 +1,145 @@ +/* Copyright 2020-2022 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package library + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "log" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// SplitVersionFromStack takes a stack/version tag and splits the stack name from the version +func SplitVersionFromStack(stackWithVersion string) (string, string, error) { + var requestVersion string + var stack string + + if valid, err := ValidateStackVersionTag(stackWithVersion); !valid { + if err != nil { + return "", "", err + } + return "", "", fmt.Errorf("stack/version tag '%s' is malformed, use form ':' or ''", + stackWithVersion) + } else if strings.Contains(stackWithVersion, ":") { + pair := strings.Split(stackWithVersion, ":") + if len(pair) != 2 { + return "", "", fmt.Errorf("problem splitting stack/version pair from tag '%s', instead of a pair got a length of %d", + stackWithVersion, len(pair)) + } + + stack = pair[0] + requestVersion = pair[1] + } else { + stack = stackWithVersion + requestVersion = "" + } + + return stack, requestVersion, nil +} + +// ValidateStackVersionTag returns true if stack/version tag is well formed +// and false if it is malformed +func ValidateStackVersionTag(stackWithVersion string) (bool, error) { + const exp = `^[a-z][^:\s]*(:([a-z]|[0-9])[^:\s]*)?$` + r, err := regexp.Compile(exp) + if err != nil { + return false, err + } + + return r.MatchString(stackWithVersion), nil +} + +// decompress extracts the archive file +func decompress(targetDir string, tarFile string, excludeFiles []string) error { + reader, err := os.Open(tarFile) + if err != nil { + return err + } + defer reader.Close() + + gzReader, err := gzip.NewReader(reader) + if err != nil { + return err + } + defer gzReader.Close() + + tarReader := tar.NewReader(gzReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + if isExcluded(header.Name, excludeFiles) { + continue + } + + target := path.Join(targetDir, header.Name) + switch header.Typeflag { + case tar.TypeDir: + err = os.MkdirAll(target, os.FileMode(header.Mode)) + if err != nil { + return err + } + case tar.TypeReg: + w, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + _, err = io.Copy(w, tarReader) + if err != nil { + return err + } + w.Close() + default: + log.Printf("Unsupported type: %v", header.Typeflag) + } + } + + return nil +} + +func isExcluded(name string, excludeFiles []string) bool { + basename := filepath.Base(name) + for _, excludeFile := range excludeFiles { + if basename == excludeFile { + return true + } + } + return false +} + +//setHeaders sets the request headers +func setHeaders(headers *http.Header, options RegistryOptions) { + t := options.Telemetry + if t.User != "" { + headers.Add("User", t.User) + } + if t.Client != "" { + headers.Add("Client", t.Client) + } + if t.Locale != "" { + headers.Add("Locale", t.Locale) + } +} diff --git a/registry-library/library/util_test.go b/registry-library/library/util_test.go new file mode 100644 index 000000000..c489916ce --- /dev/null +++ b/registry-library/library/util_test.go @@ -0,0 +1,117 @@ +package library + +import ( + "reflect" + "testing" +) + +func TestValidateStackVersionTag(t *testing.T) { + tests := []struct { + name string + stackTag string + wantResult bool + wantErr bool + }{ + { + name: "Stack without version", + stackTag: "java-maven", + wantResult: true, + }, + { + name: "Stack with specific version", + stackTag: "java-maven:1.0.0", + wantResult: true, + }, + { + name: "Stack with labelled version", + stackTag: "java-maven:latest", + wantResult: true, + }, + { + name: "Invalid stack name", + stackTag: "134stack", + wantResult: false, + }, + { + name: "Invalid version", + stackTag: "java-maven:.", + wantResult: false, + }, + { + name: "Malformed stack tag", + stackTag: "36g::test:", + wantResult: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotResult, err := ValidateStackVersionTag(test.stackTag) + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if !reflect.DeepEqual(gotResult, test.wantResult) { + t.Errorf("Expected: %+v, \nGot: %+v", test.wantResult, gotResult) + } + }) + } +} + +func TestSplitVersionFromStack(t *testing.T) { + tests := []struct { + name string + stackTag string + wantStack string + wantVersion string + wantErr bool + }{ + { + name: "Stack without version", + stackTag: "java-maven", + wantStack: "java-maven", + }, + { + name: "Stack with specific version", + stackTag: "java-maven:1.0.0", + wantStack: "java-maven", + wantVersion: "1.0.0", + }, + { + name: "Stack with labelled version", + stackTag: "java-maven:latest", + wantStack: "java-maven", + wantVersion: "latest", + }, + { + name: "Invalid stack name", + stackTag: "134stack", + wantErr: true, + }, + { + name: "Invalid version", + stackTag: "java-maven:.", + wantErr: true, + }, + { + name: "Malformed stack tag", + stackTag: "36g::test:", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotStack, gotVersion, err := SplitVersionFromStack(test.stackTag) + if !test.wantErr && err != nil { + t.Errorf("Unexpected err: %+v", err) + } else if test.wantErr && err == nil { + t.Error("Expected error but got nil") + } else if !reflect.DeepEqual(gotStack, test.wantStack) { + t.Errorf("Expected Stack Name: %+v, \nGot Stack Name: %+v", test.wantStack, gotStack) + } else if !reflect.DeepEqual(gotVersion, test.wantVersion) { + t.Errorf("Expected Version: %+v, \nGot Version: %+v", test.wantVersion, gotVersion) + } + }) + } +} diff --git a/registry-library/vendor/github.com/devfile/registry-support/index/generator/schema/schema.go b/registry-library/vendor/github.com/devfile/registry-support/index/generator/schema/schema.go index 21c5db9ab..8f7ec9ab6 100644 --- a/registry-library/vendor/github.com/devfile/registry-support/index/generator/schema/schema.go +++ b/registry-library/vendor/github.com/devfile/registry-support/index/generator/schema/schema.go @@ -157,16 +157,16 @@ type StarterProject struct { type Devfile struct { Meta Schema `yaml:"metadata,omitempty" json:"metadata,omitempty"` StarterProjects []StarterProject `yaml:"starterProjects,omitempty" json:"starterProjects,omitempty"` - SchemaVersion string `yaml:"schemaVersion,omitempty" json:"schemaVersion,omitempty"` + SchemaVersion string `yaml:"schemaVersion,omitempty" json:"schemaVersion,omitempty"` } // Git stores the information of remote repositories type Git struct { - Remotes map[string]string `yaml:"remotes,omitempty" json:"remotes,omitempty"` - Url string `yaml:"url,omitempty" json:"url,omitempty"` - RemoteName string `yaml:"remoteName,omitempty" json:"remoteName,omitempty"` - SubDir string `yaml:"subDir,omitempty" json:"subDir,omitempty"` - Revision string `yaml:"revision,omitempty" json:"revision,omitempty"` + Remotes map[string]string `yaml:"remotes,omitempty" json:"remotes,omitempty"` + Url string `yaml:"url,omitempty" json:"url,omitempty"` + RemoteName string `yaml:"remoteName,omitempty" json:"remoteName,omitempty"` + SubDir string `yaml:"subDir,omitempty" json:"subDir,omitempty"` + Revision string `yaml:"revision,omitempty" json:"revision,omitempty"` } // ExtraDevfileEntries is the extraDevfileEntries structure that is used by index component @@ -177,24 +177,24 @@ type ExtraDevfileEntries struct { // Version stores the top-level stack information defined within stack.yaml type StackInfo struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - DisplayName string `yaml:"displayName,omitempty" json:"displayName,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Icon string `yaml:"icon,omitempty" json:"icon,omitempty"` - Versions []Version `yaml:"versions,omitempty" json:"versions,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + DisplayName string `yaml:"displayName,omitempty" json:"displayName,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Icon string `yaml:"icon,omitempty" json:"icon,omitempty"` + Versions []Version `yaml:"versions,omitempty" json:"versions,omitempty"` } // Version stores the information for each stack version type Version struct { - Version string `yaml:"version,omitempty" json:"version,omitempty"` - SchemaVersion string `yaml:"schemaVersion,omitempty" json:"schemaVersion,omitempty"` - Default bool `yaml:"default,omitempty" json:"default,omitempty"` - Git *Git `yaml:"git,omitempty" json:"git,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Tags []string `yaml:"tags,omitempty" json:"tags,omitempty"` - Architectures []string `yaml:"architectures,omitempty" json:"architectures,omitempty"` - Icon string `yaml:"icon,omitempty" json:"icon,omitempty"` - Links map[string]string `yaml:"links,omitempty" json:"links,omitempty"` - Resources []string `yaml:"resources,omitempty" json:"resources,omitempty"` - StarterProjects []string `yaml:"starterProjects,omitempty" json:"starterProjects,omitempty"` -} \ No newline at end of file + Version string `yaml:"version,omitempty" json:"version,omitempty"` + SchemaVersion string `yaml:"schemaVersion,omitempty" json:"schemaVersion,omitempty"` + Default bool `yaml:"default,omitempty" json:"default,omitempty"` + Git *Git `yaml:"git,omitempty" json:"git,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Tags []string `yaml:"tags,omitempty" json:"tags,omitempty"` + Architectures []string `yaml:"architectures,omitempty" json:"architectures,omitempty"` + Icon string `yaml:"icon,omitempty" json:"icon,omitempty"` + Links map[string]string `yaml:"links,omitempty" json:"links,omitempty"` + Resources []string `yaml:"resources,omitempty" json:"resources,omitempty"` + StarterProjects []string `yaml:"starterProjects,omitempty" json:"starterProjects,omitempty"` +} diff --git a/registry-library/vendor/modules.txt b/registry-library/vendor/modules.txt index e96a5a01f..fd8ac8514 100644 --- a/registry-library/vendor/modules.txt +++ b/registry-library/vendor/modules.txt @@ -23,7 +23,7 @@ github.com/containerd/containerd/remotes/docker/auth github.com/containerd/containerd/remotes/docker/schema1 github.com/containerd/containerd/remotes/errors github.com/containerd/containerd/version -# github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e +# github.com/devfile/registry-support/index/generator v0.0.0-20220624203950-e7282a4695b6 ## explicit github.com/devfile/registry-support/index/generator/schema # github.com/docker/cli v20.10.11+incompatible diff --git a/tests/integration/pkg/tests/devfilelibrary_tests.go b/tests/integration/pkg/tests/devfilelibrary_tests.go index adab478ae..5976ce1ed 100644 --- a/tests/integration/pkg/tests/devfilelibrary_tests.go +++ b/tests/integration/pkg/tests/devfilelibrary_tests.go @@ -30,12 +30,15 @@ import ( ) const ( - nodejsStack = "nodejs" - javaMavenStack = "java-maven" - quarkusStack = "java-quarkus" - nodejsSample = "nodejs-basic" - quarkusSample = "code-with-quarkus" - pythonSample = "python-basic" + goStack = "go" + nodejsStack = "nodejs" + javaMavenStack = "java-maven" + quarkusStack = "java-quarkus" + nodejsSample = "nodejs-basic" + quarkusSample = "code-with-quarkus" + pythonSample = "python-basic" + javaMavenStarter = "springbootproject" + goStarter = "go-starter" ) var ( @@ -124,4 +127,22 @@ var _ = ginkgo.Describe("[Verify registry library works with registry]", func() _, _, err = devfilePkg.ParseDevfileAndValidate(parserArgs) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + + ginkgo.It("should properly download stack starter project", func() { + tempDir := os.TempDir() + util.CmdShouldPass("registry-library", "download", publicDevfileRegistry, javaMavenStack, javaMavenStarter, "--context", tempDir) + starterPath := path.Join(tempDir, javaMavenStarter) + info, err := os.Stat(starterPath) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(info.IsDir()).To(gomega.Equal(true)) + }) + + ginkgo.It("should properly download V2 stack starter project", func() { + tempDir := os.TempDir() + util.CmdShouldPass("registry-library", "download", publicDevfileRegistry, goStack, goStarter, "--context", tempDir, "--new-index-schema") + starterPath := path.Join(tempDir, goStarter) + info, err := os.Stat(starterPath) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(info.IsDir()).To(gomega.Equal(true)) + }) })