Adjusted IOCTypes usage. (opensearch-project#1156)

* Removed TODOs. Signed-off-by: AWSHurneyt <[email protected]> * Refactored how STIX2IOCGenerator creates IOCs of specific types. Signed-off-by: AWSHurneyt <[email protected]> * Added additional integration tests. Signed-off-by: AWSHurneyt <[email protected]> * Changed IOCType usage as it's no longer an enum in SA commons. Signed-off-by: AWSHurneyt <[email protected]> * Updated jar file. Signed-off-by: AWSHurneyt <[email protected]> * Fixed unit tests. Signed-off-by: AWSHurneyt <[email protected]> * Fixed tests. Signed-off-by: AWSHurneyt <[email protected]> * Refactored build.gradle to exclude redundant dependencies from SA commons, instead of the SA commons jar being generated withhout those dependencies. Signed-off-by: AWSHurneyt <[email protected]> * Updated jar. Signed-off-by: AWSHurneyt <[email protected]> * Reverted changes to build.gradle. Signed-off-by: AWSHurneyt <[email protected]> * Updated jar. Signed-off-by: AWSHurneyt <[email protected]> * Fixed tests. Signed-off-by: AWSHurneyt <[email protected]> * Fixed tests. Signed-off-by: AWSHurneyt <[email protected]> * Fixed tests. Signed-off-by: AWSHurneyt <[email protected]> * Fixed IOCType usage. Signed-off-by: AWSHurneyt <[email protected]> * Fixed log message. Signed-off-by: AWSHurneyt <[email protected]> * Fixed tests. Signed-off-by: AWSHurneyt <[email protected]> * Addressed PR feedback. Signed-off-by: AWSHurneyt <[email protected]> --------- Signed-off-by: AWSHurneyt <[email protected]>
AWSHurneyt · Jul 12, 2024 · e258e7f · e258e7f
1 parent f047503
commit e258e7f
Show file tree

Hide file tree

Showing 19 changed files with 528 additions and 139 deletions.
diff --git a/security-analytics-commons-1.0.0.jar b/security-analytics-commons-1.0.0.jar
diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java
@@ -75,7 +75,6 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
             xcp.nextToken();
 
             switch (fieldName) {
-                // synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
                 case STIX2.ID_FIELD:
                     if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
                         id = xcp.text();
@@ -90,7 +89,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
                     name = xcp.text();
                     break;
                 case STIX2.TYPE_FIELD:
-                    type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
+                    type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
                     break;
                 case STIX2.VALUE_FIELD:
                     value = xcp.text();
@@ -177,7 +176,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.startObject()
                 .field(STIX2IOC.ID_FIELD, ioc.getId())
                 .field(STIX2IOC.NAME_FIELD, ioc.getName())
-                .field(STIX2IOC.TYPE_FIELD, ioc.getType())
+                .field(STIX2IOC.TYPE_FIELD, ioc.getType().toString())
                 .field(STIX2IOC.VALUE_FIELD, ioc.getValue())
                 .field(STIX2IOC.SEVERITY_FIELD, ioc.getSeverity())
                 .timeField(STIX2IOC.CREATED_FIELD, ioc.getCreated())

diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java
@@ -84,7 +84,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
         this(
                 sin.readString(), // id
                 sin.readString(), // name
-                sin.readEnum(IOCType.class), // type
+                new IOCType(sin.readString()), // type
                 sin.readString(), // value
                 sin.readString(), // severity
                 sin.readInstant(), // created
@@ -142,7 +142,7 @@ public static STIX2IOC readFrom(StreamInput sin) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(super.getId());
         out.writeString(super.getName());
-        out.writeEnum(super.getType());
+        out.writeString(super.getType().toString());
         out.writeString(super.getValue());
         out.writeString(super.getSeverity());
         out.writeInstant(super.getCreated());
@@ -160,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject()
                 .field(ID_FIELD, super.getId())
                 .field(NAME_FIELD, super.getName())
-                .field(TYPE_FIELD, super.getType())
+                .field(TYPE_FIELD, super.getType().toString())
                 .field(VALUE_FIELD, super.getValue())
                 .field(SEVERITY_FIELD, super.getSeverity());
         XContentUtils.buildInstantAsField(builder, super.getCreated(), CREATED_FIELD);
@@ -205,7 +205,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
                     name = xcp.text();
                     break;
                 case TYPE_FIELD:
-                    type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
+                    type = new IOCType(xcp.text());
                     break;
                 case VALUE_FIELD:
                     value = xcp.text();
@@ -292,8 +292,8 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
     public void validate() throws IllegalArgumentException {
         if (super.getType() == null) {
             throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
-        } else if (!Arrays.asList(IOCType.values()).contains(super.getType())) {
-            logger.debug("Unsupported IOCType: {}", super.getType());
+        } else if (!IOCType.supportedType(super.getType().toString())) {
+            logger.debug("Unsupported IOCType: {}", super.getType().toString());
             throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
         }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java
@@ -102,7 +102,7 @@ public static STIX2IOCDto readFrom(StreamInput sin) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
         out.writeString(name);
-        out.writeEnum(type);
+        out.writeString(type.toString());
         out.writeString(value);
         out.writeString(severity);
         out.writeInstant(created);
@@ -120,7 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.startObject()
                 .field(STIX2IOC.ID_FIELD, id)
                 .field(STIX2IOC.NAME_FIELD, name)
-                .field(STIX2IOC.TYPE_FIELD, type)
+                .field(STIX2IOC.TYPE_FIELD, type.toString())
                 .field(STIX2IOC.VALUE_FIELD, value)
                 .field(STIX2IOC.SEVERITY_FIELD, severity)
                 .timeField(STIX2IOC.CREATED_FIELD, created)
@@ -161,7 +161,6 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
             xcp.nextToken();
 
             switch (fieldName) {
-                // synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
                 case STIX2.ID_FIELD:
                     if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
                         id = xcp.text();
@@ -176,7 +175,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
                     name = xcp.text();
                     break;
                 case STIX2.TYPE_FIELD:
-                    type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
+                    type = new IOCType(xcp.text());
                     break;
                 case STIX2.VALUE_FIELD:
                     value = xcp.text();

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java
@@ -6,6 +6,8 @@
 package org.opensearch.securityanalytics.services;
 
 import com.amazonaws.services.s3.AmazonS3;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.securityanalytics.commons.connector.Connector;
 import org.opensearch.securityanalytics.commons.connector.S3Connector;
 import org.opensearch.securityanalytics.commons.connector.codec.InputCodec;
@@ -21,6 +23,7 @@
 import java.util.List;
 
 public class STIX2IOCConnectorFactory extends UnaryParameterCachingFactory<FeedConfiguration, Connector<STIX2>> {
+    private static final Logger logger = LogManager.getLogger(STIX2IOCConnectorFactory.class);
     private final InputCodecFactory inputCodecFactory;
     private final S3ClientFactory s3ClientFactory;
 
@@ -31,7 +34,7 @@ public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final
 
     protected Connector<STIX2> doCreate(FeedConfiguration feedConfiguration) {
         final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration);
-        // TODO hurneyt add debug log for which location gets used
+        logger.debug("FeedLocation: {}", feedLocation);
         switch(feedLocation) {
             case S3: return createS3Connector(feedConfiguration);
             default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java
@@ -42,11 +42,10 @@ public void accept(final STIX2 ioc) {
                 feedStore.getSaTifSourceConfig().getName()
         );
 
-        // TODO hurneyt refactor once the enum values are updated
         // If the IOC received is not a type listed for the config, do not add it to the queue
-        if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().name())) {
+        if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) {
             log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
-                    stix2IOC.getType().name(), feedStore.getSaTifSourceConfig().getId(),
+                    stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(),
                     stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
             );
             return;

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
@@ -57,16 +57,13 @@ public class STIX2IOCFeedStore implements FeedStore {
     public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER;
 
     private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class);
+
     Instant startTime = Instant.now();
 
     private Client client;
     private ClusterService clusterService;
     private SATIFSourceConfig saTifSourceConfig;
-
-    // TODO hurneyt FetchIocsActionResponse is just a placeholder response type for now
     private ActionListener<STIX2IOCFetchService.STIX2IOCFetchResponse> baseListener;
-
-    // TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
     private Integer batchSize;
 
     public STIX2IOCFeedStore(
@@ -97,7 +94,6 @@ public void storeIOCs(Map<IOC, UpdateAction> actionToIOCs) {
         for (Map.Entry<UpdateAction, List<STIX2IOC>> entry : iocsSortedByAction.entrySet()) {
             switch (entry.getKey()) {
                 case DELETE:
-                    // TODO hurneyt consider whether DELETE actions should be handled elsewhere
                     break;
                 case UPSERT:
                     try {
@@ -119,7 +115,7 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
         initFeedIndex(newActiveIndex, ActionListener.wrap(
                 r -> {
                     saTifSourceConfig.getIocTypes().forEach(type -> {
-                        IOCType iocType = IOCType.fromString(type);
+                        IOCType iocType = new IOCType(type);
                         if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
                             List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
                                     ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java
@@ -78,7 +78,6 @@ public class STIX2IOCFetchService {
     private STIX2IOCConnectorFactory connectorFactory;
     private S3ClientFactory s3ClientFactory;
 
-    // TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
     private Integer batchSize;
     private String internalAuthEndpoint = "";
 
@@ -126,7 +125,6 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList
             listener.onFailure(e);
         }
 
-        // TODO consider passing listener into the flush IOC function
         try {
             consumer.flushIOCs();
         } catch (Exception e) {
@@ -285,15 +283,15 @@ private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SA
             String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null);
             Integer colNum = source.getCsvIocValueColumnNo();
             String iocValue = record.values()[colNum].split(" ")[0];
-            if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) {
+            if (iocType.equalsIgnoreCase(IOCType.IPV4_TYPE) && !isValidIp(iocValue)) {
                 log.info("Invalid IP address, skipping this ioc record: {}", iocValue);
                 continue;
             }
             Instant now = Instant.now();
             STIX2IOC stix2IOC = new STIX2IOC(
                     UUID.randomUUID().toString(),
                     UUID.randomUUID().toString(),
-                    iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType),
+                    iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType),
                     iocValue,
                     "high",
                     now,

diff --git a/...in/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/...in/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java
@@ -5,6 +5,7 @@
 
 package org.opensearch.securityanalytics.threatIntel.common;
 
+import org.opensearch.securityanalytics.commons.model.IOC;
 import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
 import org.opensearch.securityanalytics.threatIntel.model.S3Source;
@@ -22,15 +23,12 @@
 public class SourceConfigDtoValidator {
     public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) {
         List<String> errorMsgs = new ArrayList<>();
-        List<String> iocTypeEnumNames = Arrays.stream(IOCType.values())
-                .map(Enum::name)
-                .collect(Collectors.toList());
 
         if (sourceConfigDto.getIocTypes().isEmpty()) {
             errorMsgs.add("Must specify at least one IOC type");
         } else {
             for (String s: sourceConfigDto.getIocTypes()) {
-                if (false == iocTypeEnumNames.contains(s)) {
+                if (!IOCType.supportedType(s)) {
                     errorMsgs.add("Invalid IOC type: " + s);
                 }
             }

diff --git a/...n/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/...n/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java
@@ -41,6 +41,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -375,9 +376,9 @@ private static SearchRequest getSearchRequestForIocType(List<String> indices, St
         SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0]));
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         // add the iocs sublist
-        boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist));
+        boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD + ".keyword", iocsSublist));
         // add ioc type filter
-        boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase()));
+        boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD + ".keyword", iocType.toLowerCase(Locale.ROOT)));
         searchRequest.source().query(boolQueryBuilder);
         return searchRequest;
     }

diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java
@@ -101,21 +101,23 @@ public IocToIndexDetails(IOCType iocType, String indexPattern, String activeInde
         }
 
         public IocToIndexDetails(StreamInput sin) throws IOException {
-            this(sin.readEnum(IOCType.class),
+            this(
+                    new IOCType(sin.readString()),
                     sin.readString(),
-                    sin.readString());
+                    sin.readString()
+            );
         }
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeEnum(iocType);
+            out.writeString(iocType.toString());
             out.writeString(indexPattern);
             out.writeString(activeIndex);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             return builder.startObject()
-                    .field(IOC_TYPE_FIELD, iocType)
+                    .field(IOC_TYPE_FIELD, iocType.toString())
                     .field(INDEX_PATTERN_FIELD, indexPattern)
                     .field(ACTIVE_INDEX_FIELD, activeIndex)
                     .endObject();
@@ -150,7 +152,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
 
         public static IOCType toIocType(String name) {
             try {
-                return IOCType.fromString(name);
+                return new IOCType(name);
             } catch (IllegalArgumentException e) {
                 log.error("Invalid Ioc type, cannot be parsed.", e);
                 return null;

diff --git a/...opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/...opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java
@@ -114,7 +114,7 @@ public void onFailure(Exception e) {
                 Instant now = Instant.now();
                 String iocType = null;
                 if (tifMetadata.getIocType().equalsIgnoreCase("ip")) {
-                    iocType = IOCType.ipv4_addr.toString();
+                    iocType = IOCType.IPV4_TYPE;
                 }
                 satifSourceConfigManagementService.createOrUpdateTifSourceConfig(
                         new SATIFSourceConfigDto(

diff --git a/.../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/.../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
@@ -27,6 +27,7 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.model.STIX2IOC;
 import org.opensearch.securityanalytics.model.STIX2IOCDto;
 import org.opensearch.securityanalytics.services.STIX2IOCFetchService;
@@ -200,12 +201,12 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
                 List<STIX2IOC> validStix2IocList = new ArrayList<>();
                 // If the IOC received is not a type listed for the config, do not add it to the queue
                 for (STIX2IOC stix2IOC : stix2IOCList) {
-                    if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().name())) {
+                    if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
                         validStix2IocList.add(stix2IOC);
                     } else {
                         log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
-                                stix2IOC.getType().name(), saTifSourceConfig.getId(),
-                                stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
+                                stix2IOC.getType().toString(), saTifSourceConfig.getId(),
+                                stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
                         );
                     }
                 }
@@ -354,7 +355,7 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
                                     Set<String> concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse);
 
                                     // remove ioc types not specified in list
-                                    defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name()));
+                                    defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));
 
                                     // get the active indices
                                     defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex()));
@@ -467,7 +468,7 @@ private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> l
                                 if (newIocStoreConfig instanceof DefaultIocStoreConfig) {
                                     DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig;
                                     // remove ioc types not specified in list
-                                    defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name()));
+                                    defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));
                                     updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig);
                                 }
                                 // Update source config as succeeded, change state back to available

diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java
@@ -2889,7 +2889,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig(
             schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS);
         }
         if (iocStoreConfig == null) {
-            iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.domain_name, "indexPattern", "writeIndex")));
+            iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex")));
         }
         if (iocTypes == null) {
             iocTypes = List.of("ip");