diff --git a/pkg/database/flush.go b/pkg/database/flush.go index ad4a912de84..56e42715b2c 100644 --- a/pkg/database/flush.go +++ b/pkg/database/flush.go @@ -17,22 +17,25 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) - func (c *Client) StartFlushScheduler(config *csconfig.FlushDBCfg) (*gocron.Scheduler, error) { maxItems := 0 maxAge := "" + if config.MaxItems != nil && *config.MaxItems <= 0 { return nil, errors.New("max_items can't be zero or negative number") } + if config.MaxItems != nil { maxItems = *config.MaxItems } + if config.MaxAge != nil && *config.MaxAge != "" { maxAge = *config.MaxAge } // Init & Start cronjob every minute for alerts scheduler := gocron.NewScheduler(time.UTC) + job, err := scheduler.Every(1).Minute().Do(c.FlushAlerts, maxAge, maxItems) if err != nil { return nil, fmt.Errorf("while starting FlushAlerts scheduler: %w", err) @@ -46,38 +49,48 @@ func (c *Client) StartFlushScheduler(config *csconfig.FlushDBCfg) (*gocron.Sched if err != nil { return nil, fmt.Errorf("while parsing agents cert auto-delete duration: %w", err) } + config.AgentsGC.CertDuration = &duration } + if config.AgentsGC.LoginPassword != nil { duration, err := ParseDuration(*config.AgentsGC.LoginPassword) if err != nil { return nil, fmt.Errorf("while parsing agents login/password auto-delete duration: %w", err) } + config.AgentsGC.LoginPasswordDuration = &duration } + if config.AgentsGC.Api != nil { log.Warning("agents auto-delete for API auth is not supported (use cert or login_password)") } } + if config.BouncersGC != nil { if config.BouncersGC.Cert != nil { duration, err := ParseDuration(*config.BouncersGC.Cert) if err != nil { return nil, fmt.Errorf("while parsing bouncers cert auto-delete duration: %w", err) } + config.BouncersGC.CertDuration = &duration } + if config.BouncersGC.Api != nil { duration, err := ParseDuration(*config.BouncersGC.Api) if err != nil { return nil, fmt.Errorf("while parsing bouncers api auto-delete duration: %w", err) } + config.BouncersGC.ApiDuration = &duration } + if config.BouncersGC.LoginPassword != nil { log.Warning("bouncers auto-delete for login/password auth is not supported (use cert or api)") } } + baJob, err := scheduler.Every(1).Minute().Do(c.FlushAgentsAndBouncers, config.AgentsGC, config.BouncersGC) if err != nil { return nil, fmt.Errorf("while starting FlushAgentsAndBouncers scheduler: %w", err) @@ -89,7 +102,6 @@ func (c *Client) StartFlushScheduler(config *csconfig.FlushDBCfg) (*gocron.Sched return scheduler, nil } - func (c *Client) FlushOrphans() { /* While it has only been linked to some very corner-case bug : https://github.com/crowdsecurity/crowdsec/issues/778 */ /* We want to take care of orphaned events for which the parent alert/decision has been deleted */ @@ -98,6 +110,7 @@ func (c *Client) FlushOrphans() { c.Log.Warningf("error while deleting orphan events: %s", err) return } + if eventsCount > 0 { c.Log.Infof("%d deleted orphan events", eventsCount) } @@ -109,103 +122,77 @@ func (c *Client) FlushOrphans() { c.Log.Warningf("error while deleting orphan decisions: %s", err) return } + if eventsCount > 0 { c.Log.Infof("%d deleted orphan decisions", eventsCount) } } -func (c *Client) flushBouncers(bouncersCfg *csconfig.AuthGCCfg) { - if bouncersCfg == nil { +func (c *Client) flushBouncers(authType string, duration *time.Duration) { + if duration == nil { return } - if bouncersCfg.ApiDuration != nil { - log.Debug("trying to delete old bouncers from api") + count, err := c.Ent.Bouncer.Delete().Where( + bouncer.LastPullLTE(time.Now().UTC().Add(-*duration)), + ).Where( + bouncer.AuthTypeEQ(authType), + ).Exec(c.CTX) - deletionCount, err := c.Ent.Bouncer.Delete().Where( - bouncer.LastPullLTE(time.Now().UTC().Add(-*bouncersCfg.ApiDuration)), - ).Where( - bouncer.AuthTypeEQ(types.ApiKeyAuthType), - ).Exec(c.CTX) - if err != nil { - c.Log.Errorf("while auto-deleting expired bouncers (api key): %s", err) - } else if deletionCount > 0 { - c.Log.Infof("deleted %d expired bouncers (api auth)", deletionCount) - } + if err != nil { + c.Log.Errorf("while auto-deleting expired bouncers (%s): %s", authType, err) + return } - if bouncersCfg.CertDuration != nil { - log.Debug("trying to delete old bouncers from cert") - - deletionCount, err := c.Ent.Bouncer.Delete().Where( - bouncer.LastPullLTE(time.Now().UTC().Add(-*bouncersCfg.CertDuration)), - ).Where( - bouncer.AuthTypeEQ(types.TlsAuthType), - ).Exec(c.CTX) - if err != nil { - c.Log.Errorf("while auto-deleting expired bouncers (api key): %s", err) - } else if deletionCount > 0 { - c.Log.Infof("deleted %d expired bouncers (api auth)", deletionCount) - } + if count > 0 { + c.Log.Infof("deleted %d expired bouncers (%s)", count, authType) } } -func (c *Client) flushAgents(agentsCfg *csconfig.AuthGCCfg) { - if agentsCfg == nil { +func (c *Client) flushAgents(authType string, duration *time.Duration) { + if duration == nil { return } - if agentsCfg.CertDuration != nil { - log.Debug("trying to delete old agents from cert") - - deletionCount, err := c.Ent.Machine.Delete().Where( - machine.LastHeartbeatLTE(time.Now().UTC().Add(-*agentsCfg.CertDuration)), - ).Where( - machine.Not(machine.HasAlerts()), - ).Where( - machine.AuthTypeEQ(types.TlsAuthType), - ).Exec(c.CTX) - log.Debugf("deleted %d entries", deletionCount) - if err != nil { - c.Log.Errorf("while auto-deleting expired machine (cert): %s", err) - } else if deletionCount > 0 { - c.Log.Infof("deleted %d expired machine (cert auth)", deletionCount) - } + count, err := c.Ent.Machine.Delete().Where( + machine.LastHeartbeatLTE(time.Now().UTC().Add(-*duration)), + machine.Not(machine.HasAlerts()), + machine.AuthTypeEQ(authType), + ).Exec(c.CTX) + + if err != nil { + c.Log.Errorf("while auto-deleting expired machines (%s): %s", authType, err) + return } - if agentsCfg.LoginPasswordDuration != nil { - log.Debug("trying to delete old agents from password") - - deletionCount, err := c.Ent.Machine.Delete().Where( - machine.LastHeartbeatLTE(time.Now().UTC().Add(-*agentsCfg.LoginPasswordDuration)), - ).Where( - machine.Not(machine.HasAlerts()), - ).Where( - machine.AuthTypeEQ(types.PasswordAuthType), - ).Exec(c.CTX) - log.Debugf("deleted %d entries", deletionCount) - if err != nil { - c.Log.Errorf("while auto-deleting expired machine (password): %s", err) - } else if deletionCount > 0 { - c.Log.Infof("deleted %d expired machine (password auth)", deletionCount) - } + if count > 0 { + c.Log.Infof("deleted %d expired machines (%s auth)", count, authType) } } func (c *Client) FlushAgentsAndBouncers(agentsCfg *csconfig.AuthGCCfg, bouncersCfg *csconfig.AuthGCCfg) error { log.Debug("starting FlushAgentsAndBouncers") - c.flushBouncers(bouncersCfg) - c.flushAgents(agentsCfg) + if agentsCfg != nil { + c.flushAgents(types.TlsAuthType, agentsCfg.CertDuration) + c.flushAgents(types.PasswordAuthType, agentsCfg.LoginPasswordDuration) + } + + if bouncersCfg != nil { + c.flushBouncers(types.TlsAuthType, bouncersCfg.CertDuration) + c.flushBouncers(types.ApiKeyAuthType, bouncersCfg.ApiDuration) + } return nil } func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error { - var deletedByAge int - var deletedByNbItem int - var totalAlerts int - var err error + var ( + deletedByAge int + deletedByNbItem int + totalAlerts int + err error + ) if !c.CanFlush { c.Log.Debug("a list is being imported, flushing later") @@ -215,6 +202,7 @@ func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error { c.Log.Debug("Flushing orphan alerts") c.FlushOrphans() c.Log.Debug("Done flushing orphan alerts") + totalAlerts, err = c.TotalAlerts() if err != nil { c.Log.Warningf("FlushAlerts (max items count): %s", err) @@ -222,10 +210,12 @@ func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error { } c.Log.Debugf("FlushAlerts (Total alerts): %d", totalAlerts) + if MaxAge != "" { filter := map[string][]string{ "created_before": {MaxAge}, } + nbDeleted, err := c.DeleteAlertWithFilter(filter) if err != nil { c.Log.Warningf("FlushAlerts (max age): %s", err) @@ -235,19 +225,21 @@ func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error { c.Log.Debugf("FlushAlerts (deleted max age alerts): %d", nbDeleted) deletedByAge = nbDeleted } + if MaxItems > 0 { - //We get the highest id for the alerts - //We subtract MaxItems to avoid deleting alerts that are not old enough - //This gives us the oldest alert that we want to keep - //We then delete all the alerts with an id lower than this one - //We can do this because the id is auto-increment, and the database won't reuse the same id twice + // We get the highest id for the alerts + // We subtract MaxItems to avoid deleting alerts that are not old enough + // This gives us the oldest alert that we want to keep + // We then delete all the alerts with an id lower than this one + // We can do this because the id is auto-increment, and the database won't reuse the same id twice lastAlert, err := c.QueryAlertWithFilter(map[string][]string{ "sort": {"DESC"}, "limit": {"1"}, - //we do not care about fetching the edges, we just want the id + // we do not care about fetching the edges, we just want the id "with_decisions": {"false"}, }) c.Log.Debugf("FlushAlerts (last alert): %+v", lastAlert) + if err != nil { c.Log.Errorf("FlushAlerts: could not get last alert: %s", err) return fmt.Errorf("could not get last alert: %w", err) @@ -259,7 +251,7 @@ func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error { c.Log.Debugf("FlushAlerts (max id): %d", maxid) if maxid > 0 { - //This may lead to orphan alerts (at least on MySQL), but the next time the flush job will run, they will be deleted + // This may lead to orphan alerts (at least on MySQL), but the next time the flush job will run, they will be deleted deletedByNbItem, err = c.Ent.Alert.Delete().Where(alert.IDLT(maxid)).Exec(c.CTX) if err != nil { @@ -269,11 +261,16 @@ func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error { } } } + if deletedByNbItem > 0 { - c.Log.Infof("flushed %d/%d alerts because the max number of alerts has been reached (%d max)", deletedByNbItem, totalAlerts, MaxItems) + c.Log.Infof("flushed %d/%d alerts because the max number of alerts has been reached (%d max)", + deletedByNbItem, totalAlerts, MaxItems) } + if deletedByAge > 0 { - c.Log.Infof("flushed %d/%d alerts because they were created %s ago or more", deletedByAge, totalAlerts, MaxAge) + c.Log.Infof("flushed %d/%d alerts because they were created %s ago or more", + deletedByAge, totalAlerts, MaxAge) } + return nil }