Skip to content

Commit

Permalink
Merge pull request #466 from Security-Onion-Solutions/2.4/detections-…
Browse files Browse the repository at this point in the history 
…airgap

SOC Detections - Airgap support
  • Loading branch information
@defensivedepth
defensivedepth authored May 6, 2024
2 parents 728c5b6 + 66b4006 commit e41f5f5
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 74 deletions.
103 changes: 67 additions & 36 deletions server/modules/elastalert/elastalert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ import (
var errModuleStopped = fmt.Errorf("elastalert module has stopped running")

const (
DEFAULT_AIRGAP_BASE_PATH = "/nsm/rules/detect-sigma/rulesets/"
DEFAULT_ALLOW_REGEX = ""
DEFAULT_DENY_REGEX = ""
DEFAULT_AUTO_UPDATE_ENABLED = false
DEFAULT_AIRGAP_ENABLED = false
DEFAULT_COMMUNITY_RULES_IMPORT_FREQUENCY_SECONDS = 86400
DEFAULT_SIGMA_PACKAGE_DOWNLOAD_TEMPLATE = "https://github.com/SigmaHQ/sigma/releases/latest/download/sigma_%s.zip"
DEFAULT_ELASTALERT_RULES_FOLDER = "/opt/sensoroni/elastalert"
Expand All @@ -70,6 +71,7 @@ type IOManager interface {

type ElastAlertEngine struct {
srv *server.Server
airgapBasePath string
communityRulesImportFrequencySeconds int
sigmaPackageDownloadTemplate string
elastAlertRulesFolder string
Expand All @@ -84,7 +86,7 @@ type ElastAlertEngine struct {
interm sync.Mutex
allowRegex *regexp.Regexp
denyRegex *regexp.Regexp
autoUpdateEnabled bool
airgapEnabled bool
notify bool
stateFilePath string
IOManager
Expand Down Expand Up @@ -121,11 +123,12 @@ func (e *ElastAlertEngine) Init(config module.ModuleConfig) (err error) {
e.thread = &sync.WaitGroup{}
e.interrupt = make(chan bool, 1)

e.airgapBasePath = module.GetStringDefault(config, "airgapBasePath", DEFAULT_AIRGAP_BASE_PATH)
e.communityRulesImportFrequencySeconds = module.GetIntDefault(config, "communityRulesImportFrequencySeconds", DEFAULT_COMMUNITY_RULES_IMPORT_FREQUENCY_SECONDS)
e.sigmaPackageDownloadTemplate = module.GetStringDefault(config, "sigmaPackageDownloadTemplate", DEFAULT_SIGMA_PACKAGE_DOWNLOAD_TEMPLATE)
e.elastAlertRulesFolder = module.GetStringDefault(config, "elastAlertRulesFolder", DEFAULT_ELASTALERT_RULES_FOLDER)
e.rulesFingerprintFile = module.GetStringDefault(config, "rulesFingerprintFile", DEFAULT_RULES_FINGERPRINT_FILE)
e.autoUpdateEnabled = module.GetBoolDefault(config, "autoUpdateEnabled", DEFAULT_AUTO_UPDATE_ENABLED)
e.airgapEnabled = module.GetBoolDefault(config, "airgapEnabled", DEFAULT_AIRGAP_ENABLED)
e.autoEnabledSigmaRules = module.GetStringArrayDefault(config, "autoEnabledSigmaRules", []string{"securityonion-resources+critical", "securityonion-resources+high"})

pkgs := module.GetStringArrayDefault(config, "sigmaRulePackages", []string{"core", "emerging_threats_addon"})
Expand Down Expand Up @@ -448,51 +451,50 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {

var zips map[string][]byte
var errMap map[string]error
if e.autoUpdateEnabled {

if e.airgapEnabled {
// AirGap, load the sigma packages from disk
zips, errMap = e.loadSigmaPackagesFromDisk()
} else {
// Not AirGap, download the sigma packages
zips, errMap = e.downloadSigmaPackages()
if len(errMap) != 0 {
log.WithField("errorMap", errMap).Error("something went wrong downloading sigma packages")
}

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameElastAlert,
Status: "error",
})
}
if len(errMap) != 0 {
log.WithField("errorMap", errMap).Error("something went wrong loading sigma packages")

continue
if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameElastAlert,
Status: "error",
})
}

var dirtyRepos map[string]*detections.DirtyRepo
continue
}

dirtyRepos, repoChanges, err = detections.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
if strings.Contains(err.Error(), "module stopped") {
break
}
var dirtyRepos map[string]*detections.DirtyRepo

log.WithError(err).Error("unable to update Sigma repos")
dirtyRepos, repoChanges, err = detections.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
if strings.Contains(err.Error(), "module stopped") {
break
}

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameElastAlert,
Status: "error",
})
}
log.WithError(err).Error("unable to update Sigma repos")

continue
if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameElastAlert,
Status: "error",
})
}

for k, v := range dirtyRepos {
allRepos[k] = v.Repo
}
} else {
// Possible airgap installation, or admin has disabled auto-updates.
continue
}

// TODO: Perform a one-time check for a pre-downloaded ruleset on disk and if exists,
// let the rest of the loop continue but then exit the loop. For now we're just hardcoding
// to always exit the loop.
return
for k, v := range dirtyRepos {
allRepos[k] = v.Repo
}

zipHashes := map[string]string{}
Expand Down Expand Up @@ -907,6 +909,35 @@ func (e *ElastAlertEngine) syncCommunityDetections(ctx context.Context, detects
return errMap, nil
}

func (e *ElastAlertEngine) loadSigmaPackagesFromDisk() (zipData map[string][]byte, errMap map[string]error) {
errMap = map[string]error{} // map[pkgName]error
defer func() {
if len(errMap) == 0 {
errMap = nil
}
}()

zipData = map[string][]byte{}
stats := map[string]int{}

for _, pkg := range e.sigmaRulePackages {
filePath := filepath.Join(e.airgapBasePath, "sigma_"+pkg+".zip")

data, err := e.ReadFile(filePath)
if err != nil {
errMap[pkg] = err
continue
}

zipData[pkg] = data
stats[pkg] = len(data)
}

log.WithField("packageSizes", stats).Info("loaded sigma packages from disk")

return zipData, errMap
}

func (e *ElastAlertEngine) downloadSigmaPackages() (zipData map[string][]byte, errMap map[string]error) {
errMap = map[string]error{} // map[pkgName]error
defer func() {
Expand Down
45 changes: 44 additions & 1 deletion server/modules/elastalert/elastalert_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func TestElastAlertModule(t *testing.T) {

err := mod.Init(nil)
assert.NoError(t, err)
assert.False(t, mod.autoUpdateEnabled)
assert.False(t, mod.airgapEnabled)

err = mod.Start()
assert.NoError(t, err)
Expand Down Expand Up @@ -525,6 +525,49 @@ func TestDownloadSigmaPackages(t *testing.T) {
}
}

func TestLoadSigmaPackagesFromDisks(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mio := mock.NewMockIOManager(ctrl)
airgapBasePath := "/nsm/rules/detect-sigma/rulesets/"

// List of packages to be loaded from disk
pkgs := []string{"core", "core+", "core++", "emerging_threats_addon", "all_rules", "fake"}

// Setting up mocks for each expected file read, except for the "fake" package to simulate an error
for _, pkg := range pkgs[:len(pkgs)-1] {
expectedFilePath := airgapBasePath + "sigma_" + pkg + ".zip"
//mio.EXPECT().Join(airgapBasePath, "sigma_"+pkg+".zip").Return(expectedFilePath)
mio.EXPECT().ReadFile(expectedFilePath).Return([]byte("mocked data for "+pkg), nil)
}

// Simulating an error for the 'fake' package
fakeFilePath := airgapBasePath + "sigma_fake.zip"
//mio.EXPECT().JoinPath(airgapBasePath, "sigma_fake.zip").Return(fakeFilePath)
mio.EXPECT().ReadFile(fakeFilePath).Return(nil, errors.New("file not found"))

engine := ElastAlertEngine{
sigmaRulePackages: pkgs,
airgapBasePath: airgapBasePath,
IOManager: mio,
}

zipData, errMap := engine.loadSigmaPackagesFromDisk()

// Assertions
assert.NotNil(t, errMap, "Expected error map to not be nil")
assert.Error(t, errMap["fake"], "Expected an error for 'fake' package")
assert.Nil(t, errMap["core"], "No error expected for 'core' package")
assert.Len(t, zipData, len(pkgs)-1, "Expect one less entry in zipData due to the 'fake' package error")

// Verify that the content for each successful load is as expected
for _, pkg := range pkgs[:len(pkgs)-1] {
expectedContent := []byte("mocked data for " + pkg)
assert.Equal(t, expectedContent, zipData[pkg], "Expect the content to match the mocked content for package: "+pkg)
}
}

const (
SimpleRuleSID = "bcc6f179-11cd-4111-a9a6-0fab68515cf7"
SimpleRule = `title: Griffon Malware Attack Pattern
Expand Down
4 changes: 3 additions & 1 deletion server/modules/elastalert/mock/mock_iomanager.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

60 changes: 24 additions & 36 deletions server/modules/strelka/strelka.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ const (
DEFAULT_REPOS_FOLDER = "/opt/sensoroni/yara/repos"
DEFAULT_COMPILE_YARA_PYTHON_SCRIPT_PATH = "/opt/so/conf/strelka/compile_yara.py"
DEFAULT_COMPILE_RULES = true
DEFAULT_AUTO_UPDATE_ENABLED = false
DEFAULT_STATE_FILE_PATH = "/opt/sensoroni/fingerprints/strelkaengine.state"
DEFAULT_AUTO_ENABLED_YARA_RULES = "securityonion-yara"
)
Expand Down Expand Up @@ -71,7 +70,6 @@ type StrelkaEngine struct {
allowRegex *regexp.Regexp
denyRegex *regexp.Regexp
compileRules bool
autoUpdateEnabled bool
notify bool
stateFilePath string
IOManager
Expand Down Expand Up @@ -108,7 +106,6 @@ func (e *StrelkaEngine) Init(config module.ModuleConfig) (err error) {
e.reposFolder = module.GetStringDefault(config, "reposFolder", DEFAULT_REPOS_FOLDER)
e.compileYaraPythonScriptPath = module.GetStringDefault(config, "compileYaraPythonScriptPath", DEFAULT_COMPILE_YARA_PYTHON_SCRIPT_PATH)
e.compileRules = module.GetBoolDefault(config, "compileRules", DEFAULT_COMPILE_RULES)
e.autoUpdateEnabled = module.GetBoolDefault(config, "autoUpdateEnabled", DEFAULT_AUTO_UPDATE_ENABLED)
e.autoEnabledYaraRules = module.GetStringArrayDefault(config, "autoEnabledYaraRules", []string{DEFAULT_AUTO_ENABLED_YARA_RULES})

e.rulesRepos, err = model.GetReposDefault(config, "rulesRepos", []*model.RuleRepo{
Expand Down Expand Up @@ -297,47 +294,38 @@ func (e *StrelkaEngine) startCommunityRuleImport() {

upToDate := map[string]*model.RuleRepo{}

if e.autoUpdateEnabled {
allRepos, anythingNew, err := detections.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
if strings.Contains(err.Error(), "module stopped") {
break
}
}

// If no import has been completed, then do a full sync
if lastImport == nil {
forceSync = true
allRepos, anythingNew, err := detections.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
if strings.Contains(err.Error(), "module stopped") {
break
}
}

if !anythingNew && !forceSync {
// no updates, skip
log.Info("Strelka sync found no changes")
// If no import has been completed, then do a full sync
if lastImport == nil {
forceSync = true
}

detections.WriteStateFile(e.IOManager, e.stateFilePath)
if !anythingNew && !forceSync {
// no updates, skip
log.Info("Strelka sync found no changes")

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "success",
})
}
detections.WriteStateFile(e.IOManager, e.stateFilePath)

continue
if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "success",
})
}

for k, v := range allRepos {
if v.WasModified || forceSync {
upToDate[k] = v.Repo
}
}
} else {
// Possible airgap installation, or admin has disabled auto-updates.
continue
}

// TODO: Perform a one-time check for a pre-downloaded ruleset on disk and if exists,
// let the rest of the loop continue but then exit the loop. For now we're just hardcoding
// to always exit the loop.
return
for k, v := range allRepos {
if v.WasModified || forceSync {
upToDate[k] = v.Repo
}
}

communityDetections, err := e.srv.Detectionstore.GetAllCommunitySIDs(e.srv.Context, util.Ptr(model.EngineNameStrelka))
Expand Down

0 comments on commit e41f5f5

Please sign in to comment.