diff --git a/DEVELOPING.md b/DEVELOPING.md index a6b4a414..7b30abd7 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -7,7 +7,8 @@ well as acceptance testing. You will require the following: - Python 3.8+ installed on your system. Consider using [pyenv](https://github.com/pyenv/pyenv) if you do not have a preference for managing python interpreter installations. - +- `zstd` binary utility if you are packaging v6+ DB schemas +- _(optional)_ `xz` binary utility if you have specifically overridden the package command options - [Poetry](https://python-poetry.org/) installed for dependency and virtualenv management for python dependencies, to install: diff --git a/README.md b/README.md index 121d32b2..e259d53f 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ curl -sSfL https://raw.githubusercontent.com/anchore/grype-db/main/install.sh | curl -sSfL https://raw.githubusercontent.com/anchore/grype-db/main/install.sh | sh -s -- -b ``` +> [!IMPORTANT] +> You will require the `zstd` utility installed on your system to support the `package` command. ## Usage @@ -39,6 +41,7 @@ grype-db pull [-g] [-p PROVIDER ...] grype-db build [-g] [--dir=DIR] [--schema=SCHEMA] [--skip-validation] [-p PROVIDER ...] # Package the already built DB file into an archive ready for upload and serving +# note: you will require the zstd utility to be installed on your system grype-db package [--dir=DIR] [--publish-base-url=URL] ``` @@ -54,7 +57,7 @@ is created that is used in packaging and curation of the database file by this a and a `provider-metadata.json` file is created that includes the last successful run date for each provider. Use `-g` to generate the list of providers to pull based on the output of "vunnel list". -The `package` command archives the `vulnerability.db`, `metadata.json` and `provider-metadata.json` files into a `tar.gz` file. Additionally, a `listing.json` +The `package` command archives the `vulnerability.db` file into a `tar.zstd` file. Additionally, a `latest.json` is generated to aid in serving one or more database archives for downstream consumption, where the consuming application should use the listing file to discover available archives available for download. The base URL used to create the download URL for each database archive is controlled by the `package.base-url` configuration option. diff --git a/cmd/grype-db/application/application.go b/cmd/grype-db/application/application.go index 7d17a78c..61de0cfa 100644 --- a/cmd/grype-db/application/application.go +++ b/cmd/grype-db/application/application.go @@ -22,6 +22,7 @@ import ( "github.com/anchore/grype-db/internal/log" "github.com/anchore/grype-db/internal/ui" "github.com/anchore/grype-db/internal/utils" + "github.com/anchore/grype/grype" ) const Name = internal.ApplicationName @@ -169,6 +170,7 @@ func setupLogger(app *Config) error { } log.Set(l) + grype.SetLogger(l) return nil } diff --git a/cmd/grype-db/cli/commands/build.go b/cmd/grype-db/cli/commands/build.go index 3e376187..ea19fe8c 100644 --- a/cmd/grype-db/cli/commands/build.go +++ b/cmd/grype-db/cli/commands/build.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "os" - "time" "github.com/scylladb/go-set/strset" "github.com/spf13/cobra" @@ -108,7 +107,7 @@ func runBuild(cfg buildConfig) error { return fmt.Errorf("unable to get provider states: %w", err) } - earliest, err := earliestTimestamp(states) + earliest, err := provider.States(states).EarliestTimestamp() if err != nil { return fmt.Errorf("unable to get earliest timestamp: %w", err) } @@ -152,38 +151,3 @@ func providerStates(skipValidation bool, providers []provider.Provider) ([]provi } return states, nil } - -func earliestTimestamp(states []provider.State) (time.Time, error) { - if len(states) == 0 { - return time.Time{}, fmt.Errorf("cannot find earliest timestamp: no states provided") - } - - // special case when there is exactly 1 state, return its timestamp even - // if it is nvd, because otherwise quality gates that pull only nvd deterministically fail. - if len(states) == 1 { - return states[0].Timestamp, nil - } - - var earliest time.Time - for _, s := range states { - // the NVD api is constantly down, so we don't want to consider it for the earliest timestamp - if s.Provider == "nvd" { - log.WithFields("provider", s.Provider).Debug("not considering data age for provider") - continue - } - if earliest.IsZero() { - earliest = s.Timestamp - continue - } - if s.Timestamp.Before(earliest) { - earliest = s.Timestamp - } - } - - if earliest.IsZero() { - return time.Time{}, fmt.Errorf("unable to determine earliest timestamp") - } - - log.WithFields("timestamp", earliest).Debug("earliest data timestamp") - return earliest, nil -} diff --git a/go.mod b/go.mod index e20c0f65..38ebb8ee 100644 --- a/go.mod +++ b/go.mod @@ -177,6 +177,8 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect + github.com/openvex/go-vex v0.2.5 // indirect + github.com/package-url/packageurl-go v0.1.1 // indirect github.com/pborman/indent v1.2.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect diff --git a/go.sum b/go.sum index 76ddf111..07475ece 100644 --- a/go.sum +++ b/go.sum @@ -810,6 +810,10 @@ github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bl github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/openvex/go-vex v0.2.5 h1:41utdp2rHgAGCsG+UbjmfMG5CWQxs15nGqir1eRgSrQ= +github.com/openvex/go-vex v0.2.5/go.mod h1:j+oadBxSUELkrKh4NfNb+BPo77U3q7gdKME88IO/0Wo= +github.com/package-url/packageurl-go v0.1.1 h1:KTRE0bK3sKbFKAk3yy63DpeskU7Cvs/x/Da5l+RtzyU= +github.com/package-url/packageurl-go v0.1.1/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/indent v1.2.1 h1:lFiviAbISHv3Rf0jcuh489bi06hj98JsVMtIDZQb9yM= diff --git a/internal/tarutil/writer.go b/internal/tarutil/writer.go index a22984fe..a32f2a15 100644 --- a/internal/tarutil/writer.go +++ b/internal/tarutil/writer.go @@ -2,13 +2,17 @@ package tarutil import ( "archive/tar" + "bufio" "compress/gzip" "fmt" "io" "os" + "os/exec" "strings" - "github.com/klauspost/compress/zstd" + "github.com/google/shlex" + + "github.com/anchore/grype-db/internal/log" ) var ErrUnsupportedArchiveSuffix = fmt.Errorf("archive name has an unsupported suffix") @@ -20,7 +24,7 @@ type writer struct { writer *tar.Writer } -// NewWriter creates a new tar writer that writes to the specified archive path. Supports .tar.gz and .tar.zst file extensions. +// NewWriter creates a new tar writer that writes to the specified archive path. Supports .tar.gz, .tar.zst, .tar.xz, and .tar file extensions. func NewWriter(archivePath string) (Writer, error) { w, err := newCompressor(archivePath) if err != nil { @@ -45,19 +49,88 @@ func newCompressor(archivePath string) (io.WriteCloser, error) { case strings.HasSuffix(archivePath, ".tar.gz"): return gzip.NewWriter(archive), nil case strings.HasSuffix(archivePath, ".tar.zst"): - // adding zstd.WithWindowSize(zstd.MaxWindowSize), zstd.WithAllLitEntropyCompression(true) - // will have slightly better results, but use a lot more memory - w, err := zstd.NewWriter(archive, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) - if err != nil { - return nil, fmt.Errorf("unable to get zst compression stream: %w", err) - } - return w, nil + // note: since we're using --ultra this tends to have a high memory usage at decompression time + // For ~700 MB payload that is compressing down to ~60 MB, that would need ~130 MB of memory (--ultra -22) + // for the same payload compressing down to ~65MB, that would need ~70MB of memory (--ultra -21) + return newShellCompressor("zstd -T0 -22 --ultra -c -vv", archive) + case strings.HasSuffix(archivePath, ".tar.xz"): + return newShellCompressor("xz -9 --threads=0 -c -vv", archive) case strings.HasSuffix(archivePath, ".tar"): return archive, nil } return nil, ErrUnsupportedArchiveSuffix } +// shellCompressor wraps the stdin pipe of an external compression process and ensures proper cleanup. +type shellCompressor struct { + cmd *exec.Cmd + pipe io.WriteCloser +} + +func newShellCompressor(c string, archive io.Writer) (*shellCompressor, error) { + args, err := shlex.Split(c) + if err != nil { + return nil, fmt.Errorf("unable to parse command: %w", err) + } + binary := args[0] + + binPath, err := exec.LookPath(binary) + if err != nil { + return nil, fmt.Errorf("unable to find binary %q: %w", binary, err) + } + if binPath == "" { + return nil, fmt.Errorf("unable to find binary %q in PATH", binary) + } + + args = args[1:] + cmd := exec.Command(binary, args...) + log.Debug(strings.Join(cmd.Args, " ")) + cmd.Stdout = archive + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("unable to create stderr pipe: %w", err) + } + + pipe, err := cmd.StdinPipe() + if err != nil { + return nil, fmt.Errorf("unable to create stdin pipe: %w", err) + } + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("unable to start process: %w", err) + } + + go func() { + scanner := bufio.NewScanner(stderrPipe) + for scanner.Scan() { + log.Debugf("[%s] %s", binary, scanner.Text()) + } + if err := scanner.Err(); err != nil { + log.Errorf("[%s] error reading stderr: %v", binary, err) + } + }() + + return &shellCompressor{ + cmd: cmd, + pipe: pipe, + }, nil +} + +func (sc *shellCompressor) Write(p []byte) (int, error) { + return sc.pipe.Write(p) +} + +func (sc *shellCompressor) Close() error { + if err := sc.pipe.Close(); err != nil { + return fmt.Errorf("unable to close compression stdin pipe: %w", err) + } + if err := sc.cmd.Wait(); err != nil { + return fmt.Errorf("compression process error: %w", err) + } + return nil +} + func (w *writer) WriteEntry(entry Entry) error { return entry.writeEntry(w.writer) } diff --git a/pkg/process/package.go b/pkg/process/package.go index 2d010082..2a8d369c 100644 --- a/pkg/process/package.go +++ b/pkg/process/package.go @@ -1,120 +1,143 @@ package process import ( + "errors" "fmt" - "net/url" "os" "path" + "path/filepath" "strings" "time" - "github.com/spf13/afero" + "github.com/scylladb/go-set/strset" "github.com/anchore/grype-db/internal/log" "github.com/anchore/grype-db/internal/tarutil" - "github.com/anchore/grype/grype/db/legacy/distribution" + "github.com/anchore/grype-db/pkg/provider" + grypeDBLegacyDistribution "github.com/anchore/grype/grype/db/legacy/distribution" + v6 "github.com/anchore/grype/grype/db/v6" + v6Distribution "github.com/anchore/grype/grype/db/v6/distribution" ) -func secondsSinceEpoch() int64 { - return time.Now().UTC().Unix() -} - func Package(dbDir, publishBaseURL, overrideArchiveExtension string) error { - log.WithFields("from", dbDir, "url", publishBaseURL, "extension-override", overrideArchiveExtension).Info("packaging database") + // check if metadata file exists, if so, then this + if _, err := os.Stat(filepath.Join(dbDir, grypeDBLegacyDistribution.MetadataFileName)); os.IsNotExist(err) { + return packageDB(dbDir, overrideArchiveExtension) + } + return packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension) +} - fs := afero.NewOsFs() - metadata, err := distribution.NewMetadataFromDir(fs, dbDir) +func packageDB(dbDir, overrideArchiveExtension string) error { + extension, err := resolveExtension(overrideArchiveExtension) if err != nil { return err } + log.WithFields("from", dbDir, "extension", extension).Info("packaging database") - if metadata == nil { - return fmt.Errorf("no metadata found in %q", dbDir) - } - - u, err := url.Parse(publishBaseURL) + s, err := v6.NewReader(v6.Config{DBDirPath: dbDir}) if err != nil { - return err + return fmt.Errorf("unable to open vulnerability store: %w", err) } - // we need a well-ordered string to append to the archive name to ensure uniqueness (to avoid overwriting - // existing archives in the CDN) as well as to ensure that multiple archives created in the same day are - // put in the correct order in the listing file. The DB timestamp represents the age of the data in the DB - // not when the DB was created. The trailer represents the time the DB was packaged. - trailer := fmt.Sprintf("%d", secondsSinceEpoch()) + metadata, err := s.GetDBMetadata() + if err != nil || metadata == nil { + return fmt.Errorf("unable to get vulnerability store metadata: %w", err) + } - // TODO (alex): supporting tar.zst - // var extension = "tar.zst" - var extension = "tar.gz" + if metadata.Model != v6.ModelVersion { + return fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model) + } - if overrideArchiveExtension != "" { - extension = strings.TrimLeft(overrideArchiveExtension, ".") + providerModels, err := s.AllProviders() + if err != nil { + return fmt.Errorf("unable to get all providers: %w", err) } - // TODO (alex): supporting tar.zst - // else if metadata.Version < 5 { - // extension = "tar.gz" - // } - var found bool - for _, valid := range []string{"tar.zst", "tar.gz"} { - if valid == extension { - found = true - break - } + if len(providerModels) == 0 { + return fmt.Errorf("no providers found in the vulnerability store") } - if !found { - return fmt.Errorf("invalid archive extension %q", extension) + eldest, err := toProviders(providerModels).EarliestTimestamp() + if err != nil { + return err } - // we attach a random value at the end of the file name to prevent from overwriting DBs in S3 that are already - // cached in the CDN. Ideally this would be based off of the archive checksum but a random string is simpler. + // output archive vulnerability-db_VERSION_OLDESTDATADATE_BUILTEPOCH.tar.gz, where: + // - VERSION: schema version in the form of v#.#.# + // - OLDESTDATADATE: RFC3338 formatted value of the oldest date capture date found for all contained providers + // - BUILTEPOCH: linux epoch formatted value of the database metadata built field tarName := fmt.Sprintf( - "vulnerability-db_v%d_%s_%s.%s", - metadata.Version, - metadata.Built.Format(time.RFC3339), - trailer, + "vulnerability-db_v%s_%s_%d.%s", + fmt.Sprintf("%d.%d.%d", metadata.Model, metadata.Revision, metadata.Addition), + eldest.UTC().Format(time.RFC3339), + metadata.BuildTimestamp.Unix(), extension, ) - tarPath := path.Join(dbDir, tarName) - if err := populate(tarName, dbDir); err != nil { + tarPath := filepath.Join(dbDir, tarName) + + if err := populateTar(tarPath); err != nil { return err } log.WithFields("path", tarPath).Info("created database archive") - entry, err := distribution.NewListingEntryFromArchive(fs, *metadata, tarPath, u) - if err != nil { - return fmt.Errorf("unable to create listing entry from archive: %w", err) + return writeLatestDocument(tarPath, *metadata) +} + +func toProviders(states []v6.Provider) provider.States { + var result provider.States + for _, state := range states { + result = append(result, provider.State{ + Provider: state.ID, + Timestamp: *state.DateCaptured, + }) } + return result +} - listing := distribution.NewListing(entry) - listingPath := path.Join(dbDir, distribution.ListingFileName) - if err = listing.Write(listingPath); err != nil { - return err +func resolveExtension(overrideArchiveExtension string) (string, error) { + var extension = "tar.zst" + + if overrideArchiveExtension != "" { + extension = strings.TrimLeft(overrideArchiveExtension, ".") } - log.WithFields("path", listingPath).Debug("created initial listing file") + var found bool + for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} { + if valid == extension { + found = true + break + } + } - return nil + if !found { + return "", fmt.Errorf("unsupported archive extension %q", extension) + } + return extension, nil } -func populate(tarName, dbDir string) error { +var listingFiles = strset.New("listing.json", "latest.json", "history.json") + +func populateTar(tarPath string) error { originalDir, err := os.Getwd() if err != nil { return fmt.Errorf("unable to get CWD: %w", err) } - if err = os.Chdir(dbDir); err != nil { - return fmt.Errorf("unable to cd to build dir: %w", err) - } + dbDir, tarName := filepath.Split(tarPath) - defer func() { - if err = os.Chdir(originalDir); err != nil { - log.Errorf("unable to cd to original dir: %v", err) + if dbDir != "" { + if err = os.Chdir(dbDir); err != nil { + return fmt.Errorf("unable to cd to build dir: %w", err) } - }() + + defer func() { + if err = os.Chdir(originalDir); err != nil { + log.Errorf("unable to cd to original dir: %v", err) + } + }() + } fileInfos, err := os.ReadDir("./") if err != nil { @@ -123,7 +146,7 @@ func populate(tarName, dbDir string) error { var files []string for _, fi := range fileInfos { - if fi.Name() != "listing.json" && !strings.Contains(fi.Name(), ".tar.") { + if !listingFiles.Has(fi.Name()) && !strings.Contains(fi.Name(), ".tar.") { files = append(files, fi.Name()) } } @@ -134,3 +157,29 @@ func populate(tarName, dbDir string) error { return nil } + +func writeLatestDocument(tarPath string, metadata v6.DBMetadata) error { + archive, err := v6Distribution.NewArchive(tarPath, *metadata.BuildTimestamp, metadata.Model, metadata.Revision, metadata.Addition) + if err != nil || archive == nil { + return fmt.Errorf("unable to create archive: %w", err) + } + + doc := v6Distribution.NewLatestDocument(*archive) + if doc == nil { + return errors.New("unable to create latest document") + } + + dbDir := filepath.Dir(tarPath) + + latestPath := path.Join(dbDir, v6Distribution.LatestFileName) + + fh, err := os.OpenFile(latestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("unable to create latest file: %w", err) + } + + if err = doc.Write(fh); err != nil { + return fmt.Errorf("unable to write latest document: %w", err) + } + return nil +} diff --git a/pkg/process/package_legacy.go b/pkg/process/package_legacy.go new file mode 100644 index 00000000..6672fe19 --- /dev/null +++ b/pkg/process/package_legacy.go @@ -0,0 +1,116 @@ +package process + +import ( + "fmt" + "net/url" + "path" + "path/filepath" + "strings" + "time" + + "github.com/spf13/afero" + + "github.com/anchore/grype-db/internal/log" + "github.com/anchore/grype/grype/db/legacy/distribution" + grypeDBLegacy "github.com/anchore/grype/grype/db/v5" + grypeDBLegacyStore "github.com/anchore/grype/grype/db/v5/store" +) + +func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string) error { //nolint:funlen + log.WithFields("from", dbDir, "url", publishBaseURL, "extension-override", overrideArchiveExtension).Info("packaging database") + + fs := afero.NewOsFs() + metadata, err := distribution.NewMetadataFromDir(fs, dbDir) + if err != nil { + return err + } + + if metadata == nil { + return fmt.Errorf("no metadata found in %q", dbDir) + } + + s, err := grypeDBLegacyStore.New(filepath.Join(dbDir, grypeDBLegacy.VulnerabilityStoreFileName), false) + if err != nil { + return fmt.Errorf("unable to open vulnerability store: %w", err) + } + + id, err := s.GetID() + if err != nil { + return fmt.Errorf("unable to get vulnerability store ID: %w", err) + } + + if id.SchemaVersion != metadata.Version { + return fmt.Errorf("metadata version %d does not match vulnerability store version %d", metadata.Version, id.SchemaVersion) + } + + u, err := url.Parse(publishBaseURL) + if err != nil { + return err + } + + // we need a well-ordered string to append to the archive name to ensure uniqueness (to avoid overwriting + // existing archives in the CDN) as well as to ensure that multiple archives created in the same day are + // put in the correct order in the listing file. The DB timestamp represents the age of the data in the DB + // not when the DB was created. The trailer represents the time the DB was packaged. + trailer := fmt.Sprintf("%d", secondsSinceEpoch()) + + // TODO (alex): supporting tar.zst + // var extension = "tar.zst" + var extension = "tar.gz" + + if overrideArchiveExtension != "" { + extension = strings.TrimLeft(overrideArchiveExtension, ".") + } + // TODO (alex): supporting tar.zst + // else if metadata.Version < 5 { + // extension = "tar.gz" + // } + + var found bool + for _, valid := range []string{"tar.zst", "tar.gz"} { + if valid == extension { + found = true + break + } + } + + if !found { + return fmt.Errorf("invalid archive extension %q", extension) + } + + // we attach a random value at the end of the file name to prevent from overwriting DBs in S3 that are already + // cached in the CDN. Ideally this would be based off of the archive checksum but a random string is simpler. + tarName := fmt.Sprintf( + "vulnerability-db_v%d_%s_%s.%s", + metadata.Version, + metadata.Built.Format(time.RFC3339), + trailer, + extension, + ) + tarPath := path.Join(dbDir, tarName) + + if err := populateTar(tarPath); err != nil { + return err + } + + log.WithFields("path", tarPath).Info("created database archive") + + entry, err := distribution.NewListingEntryFromArchive(fs, *metadata, tarPath, u) + if err != nil { + return fmt.Errorf("unable to create listing entry from archive: %w", err) + } + + listing := distribution.NewListing(entry) + listingPath := path.Join(dbDir, distribution.ListingFileName) + if err = listing.Write(listingPath); err != nil { + return err + } + + log.WithFields("path", listingPath).Debug("created initial listing file") + + return nil +} + +func secondsSinceEpoch() int64 { + return time.Now().UTC().Unix() +} diff --git a/pkg/provider/state.go b/pkg/provider/state.go index 81c5755f..1a04842d 100644 --- a/pkg/provider/state.go +++ b/pkg/provider/state.go @@ -134,3 +134,38 @@ func (s States) Names() []string { } return names } + +func (s States) EarliestTimestamp() (time.Time, error) { + if len(s) == 0 { + return time.Time{}, fmt.Errorf("cannot find earliest timestamp: no states provided") + } + + // special case when there is exactly 1 state, return its timestamp even + // if it is nvd, because otherwise quality gates that pull only nvd deterministically fail. + if len(s) == 1 { + return s[0].Timestamp, nil + } + + var earliest time.Time + for _, curState := range s { + // the NVD api is constantly down, so we don't want to consider it for the earliest timestamp + if curState.Provider == "nvd" { + log.WithFields("provider", curState.Provider).Debug("not considering data age for provider") + continue + } + if earliest.IsZero() { + earliest = curState.Timestamp + continue + } + if curState.Timestamp.Before(earliest) { + earliest = curState.Timestamp + } + } + + if earliest.IsZero() { + return time.Time{}, fmt.Errorf("unable to determine earliest timestamp") + } + + log.WithFields("timestamp", earliest).Debug("earliest data timestamp") + return earliest, nil +} diff --git a/cmd/grype-db/cli/commands/build_test.go b/pkg/provider/state_test.go similarity index 88% rename from cmd/grype-db/cli/commands/build_test.go rename to pkg/provider/state_test.go index 97c19af7..7544860c 100644 --- a/cmd/grype-db/cli/commands/build_test.go +++ b/pkg/provider/state_test.go @@ -1,4 +1,4 @@ -package commands +package provider import ( "reflect" @@ -6,20 +6,18 @@ import ( "time" "github.com/stretchr/testify/require" - - "github.com/anchore/grype-db/pkg/provider" ) func Test_earliestTimestamp(t *testing.T) { tests := []struct { name string - states []provider.State + states []State want time.Time wantErr require.ErrorAssertionFunc }{ { name: "happy path", - states: []provider.State{ + states: []State{ { Timestamp: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), }, @@ -34,13 +32,13 @@ func Test_earliestTimestamp(t *testing.T) { }, { name: "empty states", - states: []provider.State{}, + states: []State{}, want: time.Time{}, wantErr: requireErrorContains("cannot find earliest timestamp: no states provided"), }, { name: "single state", - states: []provider.State{ + states: []State{ { Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), }, @@ -49,7 +47,7 @@ func Test_earliestTimestamp(t *testing.T) { }, { name: "single state, but it's nvd", - states: []provider.State{ + states: []State{ { Provider: "nvd", Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), @@ -59,7 +57,7 @@ func Test_earliestTimestamp(t *testing.T) { }, { name: "all states have provider nvd", - states: []provider.State{ + states: []State{ { Provider: "nvd", Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), @@ -74,7 +72,7 @@ func Test_earliestTimestamp(t *testing.T) { }, { name: "mix of nvd and non-nvd providers", - states: []provider.State{ + states: []State{ { Provider: "nvd", Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), @@ -92,7 +90,7 @@ func Test_earliestTimestamp(t *testing.T) { }, { name: "timestamps are the same", - states: []provider.State{ + states: []State{ { Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), }, @@ -112,7 +110,7 @@ func Test_earliestTimestamp(t *testing.T) { if tt.wantErr == nil { tt.wantErr = require.NoError } - got, err := earliestTimestamp(tt.states) + got, err := States(tt.states).EarliestTimestamp() tt.wantErr(t, err) if err != nil { return