-
Notifications
You must be signed in to change notification settings - Fork 76
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Backport 2.x] New Log Type JSON format (#474)
Signed-off-by: Petar Dzepina <[email protected]> Co-authored-by: Subhobrata Dey <[email protected]>
- Loading branch information
1 parent
fa0e18e
commit 2e869e5
Showing
39 changed files
with
2,888 additions
and
633 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
109 changes: 109 additions & 0 deletions
109
src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.securityanalytics.logtype; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.net.URISyntaxException; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.function.Function; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.common.settings.SettingsException; | ||
| import org.opensearch.common.xcontent.XContentHelper; | ||
| import org.opensearch.common.xcontent.json.JsonXContent; | ||
| import org.opensearch.securityanalytics.model.LogType; | ||
| import org.opensearch.securityanalytics.util.FileUtils; | ||
|
|
||
| public class BuiltinLogTypeLoader { | ||
|
|
||
| private static final Logger logger = LogManager.getLogger(BuiltinLogTypeLoader.class); | ||
|
|
||
| private static final String BASE_PATH = "OSMapping/"; | ||
|
|
||
| private static final String LOG_TYPE_FILE_SUFFIX = "_logtype.json"; | ||
|
|
||
| private static List<LogType> logTypes; | ||
| private static Map<String, LogType> logTypeMap; | ||
|
|
||
|
|
||
| static { | ||
| ensureLogTypesLoaded(); | ||
| } | ||
|
|
||
| public static List<LogType> getAllLogTypes() { | ||
| ensureLogTypesLoaded(); | ||
| return logTypes; | ||
| } | ||
|
|
||
| public static LogType getLogTypeByName(String logTypeName) { | ||
| ensureLogTypesLoaded(); | ||
| return logTypeMap.get(logTypeName); | ||
| } | ||
|
|
||
| public static boolean logTypeExists(String logTypeName) { | ||
| ensureLogTypesLoaded(); | ||
| return logTypeMap.containsKey(logTypeName); | ||
| } | ||
|
|
||
| private static void ensureLogTypesLoaded() { | ||
| try { | ||
| if (logTypes != null) { | ||
| return; | ||
| } | ||
| logTypes = loadBuiltinLogTypes(); | ||
| logTypeMap = logTypes.stream() | ||
| .collect(Collectors.toMap(LogType::getName, Function.identity())); | ||
| } catch (Exception e) { | ||
| logger.error("Failed loading builtin log types from disk!", e); | ||
| } | ||
| } | ||
|
|
||
| private static List<LogType> loadBuiltinLogTypes() throws URISyntaxException, IOException { | ||
| List<LogType> logTypes = new ArrayList<>(); | ||
|
|
||
| final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); | ||
|
|
||
| Path dirPath = null; | ||
| if (url.contains("!")) { | ||
| final String[] paths = url.split("!"); | ||
| dirPath = FileUtils.getFs().getPath(paths[1]); | ||
| } else { | ||
| dirPath = Path.of(url); | ||
| } | ||
|
|
||
| Stream<Path> folder = Files.list(dirPath); | ||
| List<Path> logTypePaths = folder.filter(e -> e.toString().endsWith(LOG_TYPE_FILE_SUFFIX)).collect(Collectors.toList()); | ||
|
|
||
| for (Path logTypePath : logTypePaths) { | ||
| try ( | ||
| InputStream is = BuiltinLogTypeLoader.class.getResourceAsStream(logTypePath.toString()) | ||
| ) { | ||
| String logTypeFilePayload = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); | ||
|
|
||
| if (logTypeFilePayload != null) { | ||
| Map<String, Object> logTypeFileAsMap = | ||
| XContentHelper.convertToMap(JsonXContent.jsonXContent, logTypeFilePayload, false); | ||
|
|
||
| logTypes.add(new LogType(logTypeFileAsMap)); | ||
|
|
||
| logger.info("Loaded [{}] log type", logTypePath.getFileName()); | ||
| } | ||
| } catch (Exception e) { | ||
| throw new SettingsException("Failed to load builtin log types", e); | ||
| } | ||
| } | ||
|
|
||
| return logTypes; | ||
| } | ||
| } |
59 changes: 59 additions & 0 deletions
59
src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.securityanalytics.logtype; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.stream.Collectors; | ||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.securityanalytics.model.LogType; | ||
| import org.opensearch.securityanalytics.util.SecurityAnalyticsException; | ||
|
|
||
|
|
||
| /** | ||
| * | ||
| * */ | ||
| public class LogTypeService { | ||
|
|
||
| private static final Logger logger = LogManager.getLogger(LogTypeService.class); | ||
|
|
||
| private BuiltinLogTypeLoader builtinLogTypeLoader; | ||
|
|
||
| public LogTypeService() { | ||
| this.builtinLogTypeLoader = new BuiltinLogTypeLoader(); | ||
| } | ||
|
|
||
|
|
||
| public List<LogType> getAllLogTypes() { | ||
| return BuiltinLogTypeLoader.getAllLogTypes(); | ||
| } | ||
|
|
||
| public LogType getLogTypeByName(String logType) { | ||
| return BuiltinLogTypeLoader.getLogTypeByName(logType); | ||
| } | ||
|
|
||
| /** | ||
| * Returns sigmaRule rawField to ECS field mapping | ||
| * | ||
| * @param logType Log type | ||
| * @return Map of rawField to ecs field | ||
| */ | ||
| public Map<String, String> getRuleFieldMappings(String logType) { | ||
| LogType lt = getLogTypeByName(logType); | ||
|
|
||
| if (lt == null) { | ||
| throw SecurityAnalyticsException.wrap(new IllegalArgumentException("Can't get rule field mappings for invalid logType: [" + logType + "]")); | ||
| } | ||
| if (lt.getMappings() == null) { | ||
| return Map.of(); | ||
| } else { | ||
| return lt.getMappings() | ||
| .stream() | ||
| .collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs)); | ||
| } | ||
| } | ||
| } |
137 changes: 137 additions & 0 deletions
137
src/main/java/org/opensearch/securityanalytics/model/LogType.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.securityanalytics.model; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
| import org.opensearch.common.io.stream.StreamInput; | ||
| import org.opensearch.common.io.stream.StreamOutput; | ||
| import org.opensearch.common.io.stream.Writeable; | ||
| import org.opensearch.core.xcontent.ToXContentObject; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
|
|
||
| public class LogType implements Writeable { | ||
|
|
||
| private static final String ID = "id"; | ||
| private static final String NAME = "name"; | ||
| private static final String DESCRIPTION = "description"; | ||
| private static final String IS_BUILTIN = "is_builtin"; | ||
| private static final String MAPPINGS = "mappings"; | ||
| private static final String RAW_FIELD = "raw_field"; | ||
| private static final String ECS = "ecs"; | ||
| private static final String OCSF = "ocsf"; | ||
|
|
||
| private String id; | ||
| private String name; | ||
| private String description; | ||
| private Boolean isBuiltIn; | ||
| private List<Mapping> mappings; | ||
|
|
||
| public LogType(StreamInput sin) throws IOException { | ||
| this.id = sin.readString(); | ||
| this.isBuiltIn = sin.readOptionalBoolean(); | ||
| this.name = sin.readString(); | ||
| this.description = sin.readString(); | ||
| this.mappings = sin.readList(Mapping::readFrom); | ||
| } | ||
|
|
||
| public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings) { | ||
| this.id = id; | ||
| this.name = name; | ||
| this.description = description; | ||
| this.isBuiltIn = isBuiltIn; | ||
| this.mappings = mappings == null ? List.of() : mappings; | ||
| } | ||
|
|
||
| public LogType(Map<String, Object> logTypeAsMap) { | ||
| this.id = (String) logTypeAsMap.get(ID); | ||
| this.name = (String) logTypeAsMap.get(NAME); | ||
| this.description = (String) logTypeAsMap.get(DESCRIPTION); | ||
| if (logTypeAsMap.containsKey(IS_BUILTIN)) { | ||
| this.isBuiltIn = (Boolean) logTypeAsMap.get(IS_BUILTIN); | ||
| } | ||
| List<Map<String, String>> mappings = (List<Map<String, String>>)logTypeAsMap.get(MAPPINGS); | ||
| if (mappings.size() > 0) { | ||
| this.mappings = new ArrayList<>(mappings.size()); | ||
| this.mappings = mappings.stream().map(e -> | ||
| new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF)) | ||
| ).collect(Collectors.toList()); | ||
| } | ||
| } | ||
|
|
||
| public String getName() { | ||
| return name; | ||
| } | ||
|
|
||
| public String getDescription() { | ||
| return description; | ||
| } | ||
|
|
||
| public boolean getIsBuiltIn() { return isBuiltIn; } | ||
|
|
||
| public List<Mapping> getMappings() { | ||
| return mappings; | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeString(id); | ||
| out.writeOptionalBoolean(isBuiltIn); | ||
| out.writeString(name); | ||
| out.writeString(description); | ||
| out.writeCollection(mappings); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return name; | ||
| } | ||
|
|
||
| public static class Mapping implements Writeable { | ||
|
|
||
| private String rawField; | ||
| private String ecs; | ||
| private String ocsf; | ||
|
|
||
| public Mapping(StreamInput sin) throws IOException { | ||
| this.rawField = sin.readString(); | ||
| this.ecs = sin.readOptionalString(); | ||
| this.ocsf = sin.readOptionalString(); | ||
| } | ||
|
|
||
| public Mapping(String rawField, String ecs, String ocsf) { | ||
| this.rawField = rawField; | ||
| this.ecs = ecs; | ||
| this.ocsf = ocsf; | ||
| } | ||
|
|
||
| public String getRawField() { | ||
| return rawField; | ||
| } | ||
|
|
||
| public String getEcs() { | ||
| return ecs; | ||
| } | ||
|
|
||
| public String getOcsf() { | ||
| return ocsf; | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeString(rawField); | ||
| out.writeOptionalString(ecs); | ||
| out.writeOptionalString(ocsf); | ||
| } | ||
|
|
||
| public static Mapping readFrom(StreamInput sin) throws IOException { | ||
| return new Mapping(sin); | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.