new tests + item removal

cmd/crowdsec-cli/require/require.go

@@ -116,7 +116,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger)
 	}
 
 	if err := hub.Load(); err != nil {
-		return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
+		return nil, fmt.Errorf("failed to read hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
 	}
 
 	return hub, nil

pkg/cwhub/hub.go

@@ -61,7 +61,7 @@ func (h *Hub) Load() error {
 	h.logger.Debugf("loading hub idx %s", h.local.HubIndexFile)
 
 	if err := h.parseIndex(); err != nil {
-		return fmt.Errorf("failed to load hub index: %w", err)
+		return err
 	}
 
 	if err := h.localSync(); err != nil {
@@ -85,6 +85,19 @@ func (h *Hub) parseIndex() error {
 	// Iterate over the different types to complete the struct
 	for _, itemType := range ItemTypes {
 		for name, item := range h.GetItemMap(itemType) {
+			if item == nil {
+				// likely defined as empty object or null in the index file
+				return fmt.Errorf("%s:%s has no index metadata", itemType, name)
+			}
+
+			if item.RemotePath == "" {
+				return fmt.Errorf("%s:%s has no download path", itemType, name)
+			}
+
+			if (itemType == PARSERS || itemType == POSTOVERFLOWS) && item.Stage == "" {
+				return fmt.Errorf("%s:%s has no stage", itemType, name)
+			}
+
 			item.hub = h
 			item.Name = name
 

pkg/cwhub/item.go

@@ -161,6 +161,7 @@ type Item struct {
 	Content     string   `json:"content,omitempty" yaml:"-"`
 	References  []string `json:"references,omitempty" yaml:"references,omitempty"`
 
+	// NOTE: RemotePath could be derived from the other fields
 	RemotePath string                 `json:"path,omitempty" yaml:"path,omitempty"`       // path relative to the base URL eg. /parsers/stage/author/file.yaml
 	Version    string                 `json:"version,omitempty" yaml:"version,omitempty"` // the last available version
 	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`               // all the known versions
@@ -181,7 +182,7 @@ func (i *Item) InstallPath() (string, error) {
 	return SafePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
 }
 
-// downloadPath returns the location of the actual config file in the hub
+// DownloadPath returns the location of the actual config file in the hub
 // (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
 // Raises an error if the path goes outside of the hub dir.
 func (i *Item) DownloadPath() (string, error) {

pkg/cwhub/sync.go

@@ -105,6 +105,9 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo
 		fname := subsHub[2]
 
 		if ftype == PARSERS || ftype == POSTOVERFLOWS {
+			if len(subsHub) < 4 {
+				return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub))
+			}
 			stage = subsHub[1]
 			fauthor = subsHub[2]
 			fname = subsHub[3]

pkg/hubops/download.go

@@ -65,6 +65,11 @@ func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) {
 		}
 
 		if i.State.Installed {
+			// ensure the _new_ dependencies are installed too
+			if err := plan.AddCommand(NewEnableCommand(sub, c.Force)); err != nil {
+				return false, err
+			}
+
 			for _, sub2 := range disableKeys {
 				if sub2 == sub {
 					delete(toDisable, sub)
@@ -155,7 +160,7 @@ func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error {
 
 	downloaded, _, err := i.FetchContentTo(ctx, finalPath)
 	if err != nil {
-		return err
+		return fmt.Errorf("%s: %w", i.FQName(), err)
 	}
 
 	if downloaded {

pkg/hubops/plan.go

@@ -117,6 +117,7 @@ func (p *ActionPlan) Description(verbose bool) string {
 func (p *ActionPlan) verboseDescription() string {
 	sb := strings.Builder{}
 
+	// Here we display the commands in the order they will be executed.
 	for _, cmd := range p.commands {
 		sb.WriteString(colorizeOpType(cmd.OperationType()) + " " + cmd.ItemType() + ":" + cmd.Detail() + "\n")
 	}
@@ -139,7 +140,7 @@ func describe(opType string, desc map[string]map[string][]string, sb *strings.Bu
 		}
 
 		details := desc[opType][itemType]
-		// NOTE: sorting for user convenience, but it's not the order the commands will be carried out
+		// Sorting for user convenience, but it's not the same order the commands will be carried out.
 		slices.Sort(details)
 
 		if itemType != "" {
@@ -170,7 +171,7 @@ func (p *ActionPlan) compactDescription() string {
 
 	sb := strings.Builder{}
 
-	// crude hack to guarantee presentation order
+	// Enforce presentation order.
 
 	describe("download", desc, &sb)
 	delete(desc, "download")

test/bats/20_hub.bats

@@ -20,7 +20,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_strip_index
 }
 
 teardown() {

test/bats/20_hub_collections_dep.bats

@@ -20,7 +20,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_strip_index
 }
 
 teardown() {

test/bats/20_hub_items.bats

@@ -22,7 +22,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_strip_index
 }
 
 teardown() {
@@ -83,7 +82,7 @@ teardown() {
     rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics -o json
     # XXX: we are on the verbose side here...
     rune -0 jq -r ".msg" <(stderr)
-    assert_output --regexp "failed to read Hub index: failed to sync hub items: failed to scan .*: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again"
+    assert_output --regexp "failed to read hub index: failed to sync hub items: failed to scan .*: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again"
 }
 
 @test "removing or purging an item already removed by hand" {

test/bats/cscli-hubtype-install.bats

@@ -9,11 +9,6 @@ set -u
 setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
-    # XXX: reload message is different on bsd
-    # shellcheck disable=SC2089
-    RELOAD_MESSAGE="Run 'sudo systemctl reload crowdsec' for the new configuration to be effective."
-    # shellcheck disable=SC2090
-    export RELOAD_MESSAGE
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
 #    INDEX_PATH=$(config_get '.config_paths.index_path')

test/bats/cscli-hubtype-list.bats

@@ -21,7 +21,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_strip_index
 }
 
 teardown() {

test/bats/cscli-hubtype-remove.bats

@@ -10,11 +10,6 @@ set -u
 setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
-    # XXX: reload message is different on bsd
-    # shellcheck disable=SC2089
-    RELOAD_MESSAGE="Run 'sudo systemctl reload crowdsec' for the new configuration to be effective."
-    # shellcheck disable=SC2090
-    export RELOAD_MESSAGE
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
 #    INDEX_PATH=$(config_get '.config_paths.index_path')

test/bats/cscli-hubtype-upgrade.bats

@@ -12,10 +12,6 @@ set -u
 setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
-    # shellcheck disable=SC2089
-    RELOAD_MESSAGE="Run 'sudo systemctl reload crowdsec' for the new configuration to be effective."
-    # shellcheck disable=SC2090
-    export RELOAD_MESSAGE
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
     INDEX_PATH=$(config_get '.config_paths.index_path')
@@ -45,7 +41,7 @@ hub_inject_v0() {
     # add a version 0.0 to all parsers
 
     # hash of the string "v0.0"
-    sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
+    sha256_0_0="daa1832414a685d69269e0ae15024b908f4602db45f9900e9c6e7f204af207c0"
 
     new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
     echo "$new_hub" >"$INDEX_PATH"
@@ -58,7 +54,7 @@ install_v0() {
     shift
 
     cscli "$hubtype" install "$item_name"
-    echo "v0.0" > "$(jq -r '.local_path' <(cscli "$hubtype" inspect "$item_name" --no-metrics -o json))"
+    printf "%s" "v0.0" > "$(jq -r '.local_path' <(cscli "$hubtype" inspect "$item_name" --no-metrics -o json))"
 }
 
 #----------

test/bats/cscli-parsers.bats

@@ -23,7 +23,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_strip_index
 }
 
 teardown() {

test/bats/cscli-postoverflows.bats

@@ -23,7 +23,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_strip_index
 }
 
 teardown() {