From b85983e2159d82f982ed6e20f561d523b5c20cd1 Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 23 Dec 2024 23:56:48 +0100 Subject: [PATCH 1/9] wip --- cmd/crowdsec-cli/require/require.go | 2 +- pkg/cwhub/hub.go | 8 ++------ pkg/cwhub/sync.go | 10 +++++----- test/bats/20_hub_items.bats | 4 ++-- test/bats/hub-index.bats | 6 +++--- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index a44e76ae47d..f67a47efd95 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -115,7 +115,7 @@ func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) { } if err := hub.Load(); err != nil { - return nil, fmt.Errorf("failed to read hub index: %w. Run 'sudo cscli hub update' to download the index again", err) + return nil, err } return hub, nil diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 3722ceaafcd..a7055bde253 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -58,14 +58,10 @@ func (h *Hub) Load() error { h.logger.Debugf("loading hub idx %s", h.local.HubIndexFile) if err := h.parseIndex(); err != nil { - return err - } - - if err := h.localSync(); err != nil { - return fmt.Errorf("failed to sync hub items: %w", err) + return fmt.Errorf("failed to parse hub index: %w. Run 'sudo cscli hub update' to download the index again", err) } - return nil + return h.localSync() } // parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections. diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index d2b59df35d6..27b21a88fc9 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -461,13 +461,12 @@ func removeDuplicates(sl []string) []string { // localSync updates the hub state with downloaded, installed and local items. func (h *Hub) localSync() error { - err := h.syncDir(h.local.InstallDir) - if err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + if err := h.syncDir(h.local.InstallDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.InstallDir, err) } - if err = h.syncDir(h.local.HubDir); err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + if err := h.syncDir(h.local.HubDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.HubDir, err) } warnings := make([]string, 0) @@ -488,6 +487,7 @@ func (h *Hub) localSync() error { continue } + // XXX: versions should be validated in the hub parser vs := item.versionStatus() switch vs { case versionUpToDate: // latest diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 2f1c952848b..8ebe505c6e1 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -80,8 +80,8 @@ teardown() { rune -0 cscli collections install crowdsecurity/sshd rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics - # XXX: we are on the verbose side here... - assert_stderr "Error: failed to read hub index: failed to sync hub items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" + # XXX: this must be triggered during parse, not sync + assert_stderr "Error: failed to sync $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" } @test "removing or purging an item already removed by hand" { diff --git a/test/bats/hub-index.bats b/test/bats/hub-index.bats index 76759991e4a..dd5db0759d5 100644 --- a/test/bats/hub-index.bats +++ b/test/bats/hub-index.bats @@ -32,7 +32,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no index metadata." + assert_stderr --partial "failed to parse hub index: parsers:author/pars1 has no index metadata." } @test "malformed index - no download path" { @@ -46,7 +46,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no download path." + assert_stderr --partial "failed to parse hub index: parsers:author/pars1 has no download path." } @test "malformed parser - no stage" { @@ -63,7 +63,7 @@ teardown() { EOF rune -1 cscli hub list -o raw - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no stage." + assert_stderr --partial "failed to parse hub index: parsers:author/pars1 has no stage." } @test "malformed parser - short path" { From 580454df69395ca8b3b34a3f3ab93234a2915ab2 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 10:33:49 +0100 Subject: [PATCH 2/9] wip --- pkg/cwhub/hub.go | 2 +- test/bats/hub-index.bats | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index a7055bde253..d54569c077c 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -58,7 +58,7 @@ func (h *Hub) Load() error { h.logger.Debugf("loading hub idx %s", h.local.HubIndexFile) if err := h.parseIndex(); err != nil { - return fmt.Errorf("failed to parse hub index: %w. Run 'sudo cscli hub update' to download the index again", err) + return fmt.Errorf("invalid hub index: %w. Run 'sudo cscli hub update' to download the index again", err) } return h.localSync() diff --git a/test/bats/hub-index.bats b/test/bats/hub-index.bats index dd5db0759d5..a609974d67a 100644 --- a/test/bats/hub-index.bats +++ b/test/bats/hub-index.bats @@ -32,7 +32,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to parse hub index: parsers:author/pars1 has no index metadata." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no index metadata." } @test "malformed index - no download path" { @@ -46,7 +46,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to parse hub index: parsers:author/pars1 has no download path." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no download path." } @test "malformed parser - no stage" { @@ -63,7 +63,7 @@ teardown() { EOF rune -1 cscli hub list -o raw - assert_stderr --partial "failed to parse hub index: parsers:author/pars1 has no stage." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no stage." } @test "malformed parser - short path" { From ff6aba569a0e1b2765a1b5abbd1c7e1225916b14 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 13:35:43 +0100 Subject: [PATCH 3/9] tests --- pkg/cwhub/download.go | 3 + pkg/cwhub/download_test.go | 159 ++++++++++++++++++++++++++++++++++--- pkg/cwhub/fetch.go | 8 +- pkg/cwhub/sync.go | 2 +- 4 files changed, 158 insertions(+), 14 deletions(-) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 48cb2382668..6e69886524d 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,6 +12,9 @@ import ( "github.com/crowdsecurity/go-cs-lib/downloader" ) +// no need to import the lib package to use this +type NotFoundError = downloader.NotFoundError + // Downloader is used to retrieve index and items from a remote hub, with cache control. type Downloader struct { Branch string diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index fc0b257a284..8f42b817653 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -10,43 +10,182 @@ import ( "testing" "github.com/sirupsen/logrus" + logtest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestFetchIndex(t *testing.T) { ctx := context.Background() mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } if r.URL.Query().Get("with_content") == "true" { - w.WriteHeader(http.StatusOK) _, err := w.Write([]byte(`Hi I'm an index with content`)) assert.NoError(t, err) } else { - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(`Hi I'm a regular index`)) + _, err := w.Write([]byte(`Hi I'm a minified index`)) assert.NoError(t, err) } })) defer mockServer.Close() + discard := logrus.New() + discard.Out = io.Discard + downloader := &Downloader{ - Branch: "main", URLTemplate: mockServer.URL + "/%s/%s", - IndexPath: "index.txt", } - logger := logrus.New() - logger.Out = io.Discard - - destPath := filepath.Join(t.TempDir(), "index.txt") + destPath := filepath.Join(t.TempDir(), "index-here") withContent := true - downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger) + var notFoundError NotFoundError + + // bad branch + + downloader.Branch = "dev" + downloader.IndexPath = ".index.json" + + downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, discard) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // bad path + + downloader.Branch = "main" + downloader.IndexPath = "index.txt" + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // ok + + downloader.IndexPath = ".index.json" + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) require.NoError(t, err) assert.True(t, downloaded) content, err := os.ReadFile(destPath) require.NoError(t, err) assert.Equal(t, "Hi I'm an index with content", string(content)) + + // not "downloading" a second time + // since we don't have cache control in the mockServer, + // the file is downloaded to a temporary location but not replaced + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) + require.NoError(t, err) + assert.False(t, downloaded) + + // download without item content + + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err = os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, "Hi I'm a minified index", string(content)) + + // bad domain name + + downloader.URLTemplate = "x/%s/%s" + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + cstest.AssertErrorContains(t, err, `Get "x/main/.index.json": unsupported protocol scheme ""`) + assert.False(t, downloaded) + + downloader.URLTemplate = "http://x/%s/%s" + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + cstest.AssertErrorContains(t, err, `Get "http://x/main/.index.json": dial tcp: lookup x: no such host`) + assert.False(t, downloaded) + +} + +func TestFetchContent(t *testing.T) { + ctx := context.Background() + + wantContent := "{'description':'linux'}" + wantHash := "e557cb9e1cb051bc3b6a695e4396c5f8e0eff4b7b0d2cc09f7684e1d52ea2224" + remotePath := "collections/crowdsecurity/linux.yaml" + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/" + remotePath { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(wantContent)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + wantURL := mockServer.URL + "/main/collections/crowdsecurity/linux.yaml" + + // bad branch + + hubDownloader := &Downloader{ + URLTemplate: mockServer.URL + "/%s/%s", + } + + discard := logrus.New() + discard.Out = io.Discard + + destPath := filepath.Join(t.TempDir(), "content-here") + + var notFoundError NotFoundError + + // bad branch + + hubDownloader.Branch = "dev" + + downloaded, url, err := hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Empty(t, url) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // bad path + + hubDownloader.Branch = "main" + hubDownloader.IndexPath = "collections/linux.yaml" + + downloaded, url, err = hubDownloader.FetchContent(ctx, "collections/linux.yaml", destPath, wantHash, discard) + assert.Empty(t, url) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // hash mismatch: the file is not reported as downloaded because it's not replaced + + capture, hook := logtest.NewNullLogger() + capture.SetLevel(logrus.WarnLevel) + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, "1234", capture) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.False(t, downloaded) + cstest.RequireLogContains(t, hook, "hash mismatch: expected 1234, got " + wantHash) + + // ok + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err := os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, wantContent, string(content)) + + // not "downloading" a second time + // since we don't have cache control in the mockServer, + // the file is downloaded to a temporary location but not replaced + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.False(t, downloaded) } diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go index dd1a520d7e2..e8dacad4a6d 100644 --- a/pkg/cwhub/fetch.go +++ b/pkg/cwhub/fetch.go @@ -11,8 +11,8 @@ import ( ) // writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash. -// If the content is base64 encoded, it will be decoded before writing. Check for item.Content -// before calling this method. +// If the content is base64 encoded, it will be decoded before writing. Call this method only +// if item.Content if not empty. func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { if i.Content == "" { return fmt.Errorf("no embedded content for %s", i.Name) @@ -48,7 +48,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { } // FetchContentTo writes the last version of the item's YAML file to the specified path. -// Returns whether the file was downloaded, and the remote url for feedback purposes. +// If the file is embedded in the index file, it will be written directly without downloads. +// Returns whether the file was downloaded (to inform if the security engine needs reloading) +// and the remote url for feedback purposes. func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) { wantHash := i.latestHash() if wantHash == "" { diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 27b21a88fc9..321015a4f1e 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -167,6 +167,7 @@ func sortedVersions(raw []string) ([]string, error) { for idx, r := range raw { v, err := semver.NewVersion(r) if err != nil { + // TODO: should catch this during index parsing return nil, fmt.Errorf("%s: %w", r, err) } @@ -487,7 +488,6 @@ func (h *Hub) localSync() error { continue } - // XXX: versions should be validated in the hub parser vs := item.versionStatus() switch vs { case versionUpToDate: // latest From afd49a7ce7d164ac5ae9696a45b205c20b75ad21 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 14:02:42 +0100 Subject: [PATCH 4/9] hardcode remote index path to '.index.json' --- cmd/crowdsec-cli/require/require.go | 1 - pkg/cwhub/cwhub_test.go | 1 - pkg/cwhub/doc.go | 1 - pkg/cwhub/download.go | 3 +-- pkg/cwhub/download_test.go | 13 +------------ pkg/cwhub/hub_test.go | 4 ---- pkg/cwhub/itemupgrade_test.go | 3 --- 7 files changed, 2 insertions(+), 24 deletions(-) diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index f67a47efd95..dd98cd092cb 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -89,7 +89,6 @@ func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader { remote := &cwhub.Downloader{ Branch: branch, URLTemplate: urlTemplate, - IndexPath: ".index.json", } return remote diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 94a1d6ef6fd..c418820b91e 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -64,7 +64,6 @@ func testHub(t *testing.T, update bool) *Hub { indexProvider := &Downloader{ Branch: "master", URLTemplate: mockURLTemplate, - IndexPath: ".index.json", } ctx := context.Background() diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index b85d7634da4..fb7209b77ae 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -90,7 +90,6 @@ // indexProvider := cwhub.Downloader{ // URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s", // Branch: "master", -// IndexPath: ".index.json", // } // // The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 6e69886524d..fa92e9960de 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -19,7 +19,6 @@ type NotFoundError = downloader.NotFoundError type Downloader struct { Branch string URLTemplate string - IndexPath string } // IndexProvider retrieves and writes .index.json @@ -64,7 +63,7 @@ func addURLParam(rawURL string, param string, value string) (string, error) { // It uses a temporary file to avoid partial downloads, and won't overwrite the original // if it has not changed. func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) { - url, err := d.urlTo(d.IndexPath) + url, err := d.urlTo(".index.json") if err != nil { return false, fmt.Errorf("failed to build hub index request: %w", err) } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index 8f42b817653..3ca8cff616c 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -49,24 +49,14 @@ func TestFetchIndex(t *testing.T) { // bad branch downloader.Branch = "dev" - downloader.IndexPath = ".index.json" downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, discard) require.ErrorAs(t, err, ¬FoundError) assert.False(t, downloaded) - // bad path - - downloader.Branch = "main" - downloader.IndexPath = "index.txt" - - downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) - require.ErrorAs(t, err, ¬FoundError) - assert.False(t, downloaded) - // ok - downloader.IndexPath = ".index.json" + downloader.Branch = "main" downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) require.NoError(t, err) @@ -151,7 +141,6 @@ func TestFetchContent(t *testing.T) { // bad path hubDownloader.Branch = "main" - hubDownloader.IndexPath = "collections/linux.yaml" downloaded, url, err = hubDownloader.FetchContent(ctx, "collections/linux.yaml", destPath, wantHash, discard) assert.Empty(t, url) diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index c2b949b7cdf..6763d26485b 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -22,7 +22,6 @@ func TestInitHubUpdate(t *testing.T) { indexProvider := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } err = hub.Update(ctx, indexProvider, false) @@ -56,7 +55,6 @@ func TestUpdateIndex(t *testing.T) { indexProvider := &Downloader{ URLTemplate: "x", Branch: "", - IndexPath: "", } err = hub.Update(ctx, indexProvider, false) @@ -68,7 +66,6 @@ func TestUpdateIndex(t *testing.T) { indexProvider = &Downloader{ URLTemplate: "https://baddomain/crowdsecurity/%s/%s", Branch: "master", - IndexPath: ".index.json", } err = hub.Update(ctx, indexProvider, false) @@ -82,7 +79,6 @@ func TestUpdateIndex(t *testing.T) { indexProvider = &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub.local.HubIndexFile = "/does/not/exist/index.json" diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index da02837e972..3225d2f013b 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -41,7 +41,6 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub, err := NewHub(hub.local, remote, nil) @@ -101,7 +100,6 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) @@ -173,7 +171,6 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) From d9e6b1e4f3bd945bfef071c27cf01de7198aad0e Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 14:14:18 +0100 Subject: [PATCH 5/9] dry --- pkg/cwhub/item.go | 48 +++++-------------------------------------- test/bats/20_hub.bats | 4 ++-- 2 files changed, 7 insertions(+), 45 deletions(-) diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 74b1cfa3ebe..4fc0326baea 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -292,49 +292,11 @@ func (i *Item) CurrentDependencies() Dependencies { } func (i *Item) logMissingSubItems() { - if !i.HasSubItems() { - return - } - - for _, subName := range i.Parsers { - if i.hub.GetItem(PARSERS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name) - } - } - - for _, subName := range i.Scenarios { - if i.hub.GetItem(SCENARIOS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name) - } - } - - for _, subName := range i.PostOverflows { - if i.hub.GetItem(POSTOVERFLOWS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name) - } - } - - for _, subName := range i.Contexts { - if i.hub.GetItem(CONTEXTS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name) - } - } - - for _, subName := range i.AppsecConfigs { - if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) - } - } - - for _, subName := range i.AppsecRules { - if i.hub.GetItem(APPSEC_RULES, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name) - } - } - - for _, subName := range i.Collections { - if i.hub.GetItem(COLLECTIONS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) + for _, sub := range i.CurrentDependencies().byType() { + for _, subName := range sub.itemNames { + if i.hub.GetItem(sub.typeName, subName) == nil { + i.hub.logger.Errorf("can't find %s:%s, required by %s", sub.typeName, subName, i.Name) + } } } } diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 03723ecc82b..b03b58732fa 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -82,8 +82,8 @@ teardown() { new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') echo "$new_hub" >"$INDEX_PATH" rune -0 cscli hub list --error - assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb" - assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql" + assert_stderr --partial "can't find parsers:crowdsecurity/smb-logs, required by crowdsecurity/smb" + assert_stderr --partial "can't find scenarios:crowdsecurity/mysql-bf, required by crowdsecurity/mysql" } @test "loading hub reports tainted items (subitem is tainted)" { From 68063e8b91db58fbb630189c32a7c60ab21becce Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 14:24:33 +0100 Subject: [PATCH 6/9] doc --- pkg/hubops/doc.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 pkg/hubops/doc.go diff --git a/pkg/hubops/doc.go b/pkg/hubops/doc.go new file mode 100644 index 00000000000..d8dc950f0bf --- /dev/null +++ b/pkg/hubops/doc.go @@ -0,0 +1,46 @@ +/* +Package hubops is responsible for managing the local hub (items and data files) for CrowdSec. + +The index file itself (.index.json) is still managed by pkg/cwhub, which also provides the Hub +and Item structs. + +The hubops package is mostly used by cscli for the "cscli install/remove/upgrade ..." commands. + +It adopts a command-based pattern: a Plan contains a sequence of Commands. Both Plan and Command +have separate preparation and execution methods. + + - Command Interface: + The Command interface defines the contract for all operations that can be + performed on hub items. Each operation implements the Prepare and Run + methods, allowing for pre-execution setup and actual execution logic. + + - ActionPlan: + ActionPlan serves as a container for a sequence of Commands. It manages the + addition of commands, handles dependencies between them, and orchestrates their + execution. ActionPlan also provides a mechanism for interactive confirmation and dry-run. + + +To perform operations on hub items, create an ActionPlan and add the desired +Commands to it. Once all commands are added, execute the ActionPlan to perform +the operations in the correct order, handling dependencies and user confirmations. + +Example: + + hub := cwhub.NewHub(...) + plan := hubops.NewActionPlan(hub) + + downloadCmd := hubops.NewDownloadCommand(item, force) + if err := plan.AddCommand(downloadCmd); err != nil { + logrus.Fatalf("Failed to add download command: %v", err) + } + + enableCmd := hubops.NewEnableCommand(item, force) + if err := plan.AddCommand(enableCmd); err != nil { + logrus.Fatalf("Failed to add enable command: %v", err) + } + + if err := plan.Execute(ctx, confirm, dryRun, verbose); err != nil { + logrus.Fatalf("Failed to execute action plan: %v", err) + } +*/ +package hubops From 7fb6dfa1de241e3b2995b6f1d5d3b5e3c7272227 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 16:42:15 +0100 Subject: [PATCH 7/9] tests --- pkg/cwhub/cwhub_test.go | 15 +-- pkg/cwhub/hub.go | 10 +- pkg/cwhub/hub_test.go | 257 +++++++++++++++++++++++++++++++++------- 3 files changed, 227 insertions(+), 55 deletions(-) diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index c418820b91e..f26a437e0f9 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -29,10 +29,9 @@ const mockURLTemplate = "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s" var responseByPath map[string]string -// testHub initializes a temporary hub with an empty json file, optionally updating it. -func testHub(t *testing.T, update bool) *Hub { - tmpDir, err := os.MkdirTemp("", "testhub") - require.NoError(t, err) +// testHubOld initializes a temporary hub with an empty json file, optionally updating it. +func testHubOld(t *testing.T, update bool) *Hub { + tmpDir := t.TempDir() local := &csconfig.LocalHubCfg{ HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), @@ -41,7 +40,7 @@ func testHub(t *testing.T, update bool) *Hub { InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(local.HubDir, 0o700) + err := os.MkdirAll(local.HubDir, 0o700) require.NoError(t, err) err = os.MkdirAll(local.InstallDir, 0o700) @@ -53,10 +52,6 @@ func testHub(t *testing.T, update bool) *Hub { err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644) require.NoError(t, err) - t.Cleanup(func() { - os.RemoveAll(tmpDir) - }) - hub, err := NewHub(local, log.StandardLogger()) require.NoError(t, err) @@ -91,7 +86,7 @@ func envSetup(t *testing.T) *Hub { // Mock the http client HubClient.Transport = newMockTransport() - hub := testHub(t, true) + hub := testHubOld(t, true) return hub } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index d54569c077c..998a4032359 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -36,7 +36,7 @@ func (h *Hub) GetDataDir() string { // and check for unmanaged items. func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) { if local == nil { - return nil, errors.New("no hub configuration found") + return nil, errors.New("no hub configuration provided") } if logger == nil { @@ -149,12 +149,14 @@ func (h *Hub) ItemStats() []string { return ret } +var ErrUpdateAfterSync = errors.New("cannot update hub index after load/sync") + // Update downloads the latest version of the index and writes it to disk if it changed. -// It cannot be called after Load() unless the hub is completely empty. +// It cannot be called after Load() unless the index was completely empty. func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error { - if len(h.pathIndex) > 0 { + if len(h.items) > 0 { // if this happens, it's a bug. - return errors.New("cannot update hub after items have been loaded") + return ErrUpdateAfterSync } downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 6763d26485b..8707bf0d2e1 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -2,87 +2,262 @@ package cwhub import ( "context" - "fmt" + "net/http" + "net/http/httptest" "os" "testing" + "path/filepath" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func TestInitHubUpdate(t *testing.T) { - hub := envSetup(t) +// testHubCfg creates an empty hub structure in a temporary directory +// and returns its configuration object. +// +// This allow the reuse of the temporary directory / hub content for multiple instances +// of the Hub object. +func testHubCfg(t *testing.T) *csconfig.LocalHubCfg { + tempDir := t.TempDir() + + local := csconfig.LocalHubCfg{ + HubDir: filepath.Join(tempDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tempDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tempDir, "crowdsec"), + InstallDataDir: filepath.Join(tempDir, "installed-data"), + } + + err := os.MkdirAll(local.HubDir, 0o755) + require.NoError(t, err) - _, err := NewHub(hub.local, nil) + err = os.MkdirAll(local.InstallDir, 0o755) require.NoError(t, err) - ctx := context.Background() + err = os.MkdirAll(local.InstallDataDir, 0o755) + require.NoError(t, err) + + return &local +} - indexProvider := &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", +func testHub(t *testing.T, localCfg *csconfig.LocalHubCfg, indexJson string) (*Hub, error) { + if localCfg == nil { + localCfg = testHubCfg(t) } - err = hub.Update(ctx, indexProvider, false) + err := os.WriteFile(localCfg.HubIndexFile, []byte(indexJson), 0o644) require.NoError(t, err) + hub, err := NewHub(localCfg, nil) + require.NoError(t, err) err = hub.Load() + return hub, err +} + +func TestIndexEmpty(t *testing.T) { + // an empty hub is valid, and should not have warnings + hub, err := testHub(t, nil, "{}") require.NoError(t, err) + assert.Empty(t, hub.Warnings) } -func TestUpdateIndex(t *testing.T) { - // bad url template - fmt.Println("Test 'bad URL'") +func TestIndexJSON(t *testing.T) { + // but it can't be an empty string + hub, err := testHub(t, nil, "") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) - tmpIndex, err := os.CreateTemp("", "index.json") + // it must be valid json + hub, err = testHub(t, nil, "def not json") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: invalid character 'd' looking for beginning of value. Run 'sudo cscli hub update' to download the index again") + assert.Empty(t, hub.Warnings) + + hub, err = testHub(t, nil, "{") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) + + // and by json we mean an object + hub, err = testHub(t, nil, "[]") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: json: cannot unmarshal array into Go value of type cwhub.HubItems") + assert.Empty(t, hub.Warnings) +} + +func TestIndexUnknownItemType(t *testing.T) { + // Allow unknown fields in the top level object, likely new item types + hub, err := testHub(t, nil, `{"goodies": {}}`) require.NoError(t, err) + assert.Empty(t, hub.Warnings) +} + +func TestHubUpdate(t *testing.T) { + // update an empty hub with a index containing a parser. - // close the file to avoid preventing the rename on windows - err = tmpIndex.Close() + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - t.Cleanup(func() { - os.Remove(tmpIndex.Name()) - }) + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` - hub := envSetup(t) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() - hub.local.HubIndexFile = tmpIndex.Name() + ctx := context.Background() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + } + + err = hub.Update(ctx, downloader, true) + require.NoError(t, err) + + err = hub.Load() + require.NoError(t, err) + + item := hub.GetItem("parsers", "author/pars1") + assert.NotEmpty(t, item) + assert.Equal(t, "author/pars1", item.Name) +} + +func TestHubUpdateInvalidTemplate(t *testing.T) { + hub, err := testHub(t, nil, "{}") + require.NoError(t, err) ctx := context.Background() - indexProvider := &Downloader{ + downloader := &Downloader{ + Branch: "main", URLTemplate: "x", - Branch: "", } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorMessage(t, err, "failed to build hub index request: invalid URL template 'x'") +} + - // bad domain - fmt.Println("Test 'bad domain'") - indexProvider = &Downloader{ - URLTemplate: "https://baddomain/crowdsecurity/%s/%s", - Branch: "master", - } - err = hub.Update(ctx, indexProvider, false) +func TestHubUpdateCannotWrite(t *testing.T) { + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - // XXX: this is not failing - // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") - // bad target path - fmt.Println("Test 'bad target path'") + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() - indexProvider = &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", } - hub.local.HubIndexFile = "/does/not/exist/index.json" + hub.local.HubIndexFile = "/proc/foo/bar/baz/.index.json" + + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorContains(t, err, "failed to create temporary download file for /proc/foo/bar/baz/.index.json") +} + +func TestHubUpdateAfterLoad(t *testing.T) { + // Update() can't be called after Load() if the hub is not completely empty. + + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + hub, err := testHub(t, nil, index1) + require.NoError(t, err) + + index2 := ` +{ + "parsers": { + "author/pars2": { + "path": "parsers/s01-parse/pars2.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(index2)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") + err = hub.Update(ctx, downloader, true) + require.ErrorIs(t, err, ErrUpdateAfterSync) } From ce056d1829cf19936b93b134636721f9a8b17cf9 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 3 Jan 2025 16:54:53 +0100 Subject: [PATCH 8/9] tests --- pkg/cwhub/download_test.go | 3 +- pkg/cwhub/hub_test.go | 2 +- pkg/cwhub/item.go | 58 ----------------------------- pkg/cwhub/item_test.go | 25 +------------ pkg/cwhub/state.go | 61 ++++++++++++++++++++++++++++++ pkg/cwhub/state_test.go | 76 ++++++++++++++++++++++++++++++++++++++ pkg/cwhub/sync.go | 1 + 7 files changed, 142 insertions(+), 84 deletions(-) create mode 100644 pkg/cwhub/state.go create mode 100644 pkg/cwhub/state_test.go diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index 3ca8cff616c..ef1301e89e5 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -93,7 +93,8 @@ func TestFetchIndex(t *testing.T) { downloader.URLTemplate = "http://x/%s/%s" downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) - cstest.AssertErrorContains(t, err, `Get "http://x/main/.index.json": dial tcp: lookup x: no such host`) + // can be no such host, server misbehaving, etc + cstest.AssertErrorContains(t, err, `Get "http://x/main/.index.json": dial tcp: lookup x`) assert.False(t, downloaded) } diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 8707bf0d2e1..7babe095481 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -19,7 +19,7 @@ import ( // testHubCfg creates an empty hub structure in a temporary directory // and returns its configuration object. // -// This allow the reuse of the temporary directory / hub content for multiple instances +// This allow the reuse of the hub content for multiple instances // of the Hub object. func testHubCfg(t *testing.T) *csconfig.LocalHubCfg { tempDir := t.TempDir() diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 4fc0326baea..38385d9399d 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -11,8 +11,6 @@ import ( "github.com/Masterminds/semver/v3" yaml "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/crowdsec/pkg/emoji" ) const ( @@ -46,62 +44,6 @@ type ItemVersion struct { Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` } -// ItemState is used to keep the local state (i.e. at runtime) of an item. -// This data is not stored in the index, but is displayed with "cscli ... inspect". -type ItemState struct { - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` - LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` - Installed bool `json:"installed"` - Downloaded bool `json:"downloaded"` - UpToDate bool `json:"up_to_date"` - Tainted bool `json:"tainted"` - TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` -} - -// IsLocal returns true if the item has been create by a user (not downloaded from the hub). -func (s *ItemState) IsLocal() bool { - return s.Installed && !s.Downloaded -} - -// Text returns the status of the item as a string (eg. "enabled,update-available"). -func (s *ItemState) Text() string { - ret := "disabled" - - if s.Installed { - ret = "enabled" - } - - if s.IsLocal() { - ret += ",local" - } - - if s.Tainted { - ret += ",tainted" - } else if !s.UpToDate && !s.IsLocal() { - ret += ",update-available" - } - - return ret -} - -// Emoji returns the status of the item as an emoji (eg. emoji.Warning). -func (s *ItemState) Emoji() string { - switch { - case s.IsLocal(): - return emoji.House - case !s.Installed: - return emoji.Prohibited - case s.Tainted || (!s.UpToDate && !s.IsLocal()): - return emoji.Warning - case s.Installed: - return emoji.CheckMark - default: - return emoji.QuestionMark - } -} - type Dependencies struct { Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` diff --git a/pkg/cwhub/item_test.go b/pkg/cwhub/item_test.go index 703bbb5cb90..350861ff85e 100644 --- a/pkg/cwhub/item_test.go +++ b/pkg/cwhub/item_test.go @@ -6,39 +6,16 @@ import ( "github.com/stretchr/testify/require" ) -func TestItemStatus(t *testing.T) { +func TestItemStats(t *testing.T) { hub := envSetup(t) // get existing map x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) - // Get item: good and bad - for k := range x { - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - item.State.Installed = true - item.State.UpToDate = false - item.State.Tainted = false - item.State.Downloaded = true - - txt := item.State.Text() - require.Equal(t, "enabled,update-available", txt) - - item.State.Installed = true - item.State.UpToDate = false - item.State.Tainted = false - item.State.Downloaded = false - - txt = item.State.Text() - require.Equal(t, "enabled,local", txt) - } - stats := hub.ItemStats() require.Equal(t, []string{ "Loaded: 2 parsers, 1 scenarios, 3 collections", - "Unmanaged items: 3 local, 0 tainted", }, stats) } diff --git a/pkg/cwhub/state.go b/pkg/cwhub/state.go new file mode 100644 index 00000000000..518185aff1c --- /dev/null +++ b/pkg/cwhub/state.go @@ -0,0 +1,61 @@ +package cwhub + +import ( + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +// ItemState is used to keep the local state (i.e. at runtime) of an item. +// This data is not stored in the index, but is displayed with "cscli ... inspect". +type ItemState struct { + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` +} + +// IsLocal returns true if the item has been create by a user (not downloaded from the hub). +func (s *ItemState) IsLocal() bool { + return s.Installed && !s.Downloaded +} + +// Text returns the status of the item as a string (eg. "enabled,update-available"). +func (s *ItemState) Text() string { + ret := "disabled" + + if s.Installed { + ret = "enabled" + } + + if s.IsLocal() { + ret += ",local" + } + + if s.Tainted { + ret += ",tainted" + } else if !s.UpToDate && !s.IsLocal() { + ret += ",update-available" + } + + return ret +} + +// Emoji returns the status of the item as an emoji (eg. emoji.Warning). +func (s *ItemState) Emoji() string { + switch { + case s.IsLocal(): + return emoji.House + case !s.Installed: + return emoji.Prohibited + case s.Tainted || (!s.UpToDate && !s.IsLocal()): + return emoji.Warning + case s.Installed: + return emoji.CheckMark + default: + return emoji.QuestionMark + } +} diff --git a/pkg/cwhub/state_test.go b/pkg/cwhub/state_test.go new file mode 100644 index 00000000000..3ed3de16fcc --- /dev/null +++ b/pkg/cwhub/state_test.go @@ -0,0 +1,76 @@ +package cwhub + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +func TestItemStateText(t *testing.T) { + // Test the text representation of an item state. + type test struct { + state ItemState + want string + wantIcon string + } + + tests := []test{ + { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: false, + Downloaded: true, + }, + "enabled,update-available", + emoji.Warning, + }, { + ItemState{ + Installed: true, + UpToDate: true, + Tainted: false, + Downloaded: true, + }, + "enabled", + emoji.CheckMark, + }, { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: false, + Downloaded: false, + }, + "enabled,local", + emoji.House, + }, { + ItemState{ + Installed: false, + UpToDate: false, + Tainted: false, + Downloaded: true, + }, + "disabled,update-available", + emoji.Prohibited, + }, { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: true, + Downloaded: true, + }, + "enabled,tainted", + emoji.Warning, + }, + } + + for idx, tc := range tests { + t.Run("Test "+strconv.Itoa(idx), func(t *testing.T) { + got := tc.state.Text() + assert.Equal(t, tc.want, got) + assert.Equal(t, tc.wantIcon, tc.state.Emoji()) + }) + } +} diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 321015a4f1e..ee8e49f2bf0 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -108,6 +108,7 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo if len(subsHub) < 4 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub)) } + stage = subsHub[1] fauthor = subsHub[2] fname = subsHub[3] From d15582caf8eb63003066241d65e6a155f28e7dbe Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 7 Jan 2025 09:36:28 +0100 Subject: [PATCH 9/9] lint --- pkg/cwhub/cwhub_test.go | 2 +- pkg/cwhub/download_test.go | 9 +++++---- pkg/cwhub/hub_test.go | 25 ++++++++++++------------- pkg/hubops/doc.go | 1 - 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index f26a437e0f9..befd279ff65 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -62,7 +62,7 @@ func testHubOld(t *testing.T, update bool) *Hub { } ctx := context.Background() - err := hub.Update(ctx, indexProvider, false) + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index ef1301e89e5..7b0b99c28d8 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -24,6 +24,7 @@ func TestFetchIndex(t *testing.T) { if r.URL.Path != "/main/.index.json" { w.WriteHeader(http.StatusNotFound) } + if r.URL.Query().Get("with_content") == "true" { _, err := w.Write([]byte(`Hi I'm an index with content`)) assert.NoError(t, err) @@ -85,7 +86,7 @@ func TestFetchIndex(t *testing.T) { assert.Equal(t, "Hi I'm a minified index", string(content)) // bad domain name - + downloader.URLTemplate = "x/%s/%s" downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) cstest.AssertErrorContains(t, err, `Get "x/main/.index.json": unsupported protocol scheme ""`) @@ -96,7 +97,6 @@ func TestFetchIndex(t *testing.T) { // can be no such host, server misbehaving, etc cstest.AssertErrorContains(t, err, `Get "http://x/main/.index.json": dial tcp: lookup x`) assert.False(t, downloaded) - } func TestFetchContent(t *testing.T) { @@ -107,9 +107,10 @@ func TestFetchContent(t *testing.T) { remotePath := "collections/crowdsecurity/linux.yaml" mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/main/" + remotePath { + if r.URL.Path != "/main/"+remotePath { w.WriteHeader(http.StatusNotFound) } + _, err := w.Write([]byte(wantContent)) assert.NoError(t, err) })) @@ -157,7 +158,7 @@ func TestFetchContent(t *testing.T) { assert.Equal(t, wantURL, url) require.NoError(t, err) assert.False(t, downloaded) - cstest.RequireLogContains(t, hook, "hash mismatch: expected 1234, got " + wantHash) + cstest.RequireLogContains(t, hook, "hash mismatch: expected 1234, got "+wantHash) // ok diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 7babe095481..461b59de78b 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -5,8 +5,8 @@ import ( "net/http" "net/http/httptest" "os" - "testing" "path/filepath" + "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -54,6 +54,7 @@ func testHub(t *testing.T, localCfg *csconfig.LocalHubCfg, indexJson string) (*H hub, err := NewHub(localCfg, nil) require.NoError(t, err) err = hub.Load() + return hub, err } @@ -94,7 +95,6 @@ func TestIndexUnknownItemType(t *testing.T) { func TestHubUpdate(t *testing.T) { // update an empty hub with a index containing a parser. - hub, err := testHub(t, nil, "{}") require.NoError(t, err) @@ -119,7 +119,8 @@ func TestHubUpdate(t *testing.T) { if r.URL.Path != "/main/.index.json" { w.WriteHeader(http.StatusNotFound) } - _, err := w.Write([]byte(index1)) + + _, err = w.Write([]byte(index1)) assert.NoError(t, err) })) defer mockServer.Close() @@ -127,7 +128,7 @@ func TestHubUpdate(t *testing.T) { ctx := context.Background() downloader := &Downloader{ - Branch: "main", + Branch: "main", URLTemplate: mockServer.URL + "/%s/%s", } @@ -149,7 +150,7 @@ func TestHubUpdateInvalidTemplate(t *testing.T) { ctx := context.Background() downloader := &Downloader{ - Branch: "main", + Branch: "main", URLTemplate: "x", } @@ -157,9 +158,6 @@ func TestHubUpdateInvalidTemplate(t *testing.T) { cstest.RequireErrorMessage(t, err, "failed to build hub index request: invalid URL template 'x'") } - - - func TestHubUpdateCannotWrite(t *testing.T) { hub, err := testHub(t, nil, "{}") require.NoError(t, err) @@ -185,7 +183,8 @@ func TestHubUpdateCannotWrite(t *testing.T) { if r.URL.Path != "/main/.index.json" { w.WriteHeader(http.StatusNotFound) } - _, err := w.Write([]byte(index1)) + + _, err = w.Write([]byte(index1)) assert.NoError(t, err) })) defer mockServer.Close() @@ -193,7 +192,7 @@ func TestHubUpdateCannotWrite(t *testing.T) { ctx := context.Background() downloader := &Downloader{ - Branch: "main", + Branch: "main", URLTemplate: mockServer.URL + "/%s/%s", } @@ -205,7 +204,6 @@ func TestHubUpdateCannotWrite(t *testing.T) { func TestHubUpdateAfterLoad(t *testing.T) { // Update() can't be called after Load() if the hub is not completely empty. - index1 := ` { "parsers": { @@ -246,7 +244,8 @@ func TestHubUpdateAfterLoad(t *testing.T) { if r.URL.Path != "/main/.index.json" { w.WriteHeader(http.StatusNotFound) } - _, err := w.Write([]byte(index2)) + + _, err = w.Write([]byte(index2)) assert.NoError(t, err) })) defer mockServer.Close() @@ -254,7 +253,7 @@ func TestHubUpdateAfterLoad(t *testing.T) { ctx := context.Background() downloader := &Downloader{ - Branch: "main", + Branch: "main", URLTemplate: mockServer.URL + "/%s/%s", } diff --git a/pkg/hubops/doc.go b/pkg/hubops/doc.go index d8dc950f0bf..b87a42653bc 100644 --- a/pkg/hubops/doc.go +++ b/pkg/hubops/doc.go @@ -19,7 +19,6 @@ have separate preparation and execution methods. addition of commands, handles dependencies between them, and orchestrates their execution. ActionPlan also provides a mechanism for interactive confirmation and dry-run. - To perform operations on hub items, create an ActionPlan and add the desired Commands to it. Once all commands are added, execute the ActionPlan to perform the operations in the correct order, handling dependencies and user confirmations.