adds new tif source config type - url download

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -157,6 +157,7 @@
 import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction;
 import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction;
 import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction;
+import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService;
 import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
 import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService;
 import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
@@ -326,12 +327,13 @@ public Collection<Object> createComponents(Client client,
         IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry);
         ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry);
         SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
+        DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService);
         return List.of(
                 detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService,
                 mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
                 correlationAlertService, notificationService,
                 tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService,
-                ioCScanService);
+                ioCScanService, defaultTifSourceConfigLoaderService);
     }
 
     @Override

src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java

@@ -7,8 +7,11 @@
 
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.SdkClientException;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.action.bulk.BulkRequest;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.core.action.ActionListener;
@@ -29,14 +32,17 @@
 import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig;
 import org.opensearch.securityanalytics.commons.model.FeedConfiguration;
 import org.opensearch.securityanalytics.commons.model.IOCSchema;
+import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.commons.model.STIX2;
 import org.opensearch.securityanalytics.commons.model.UpdateType;
 import org.opensearch.securityanalytics.model.STIX2IOC;
 import org.opensearch.securityanalytics.model.STIX2IOCDto;
 import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
 import org.opensearch.securityanalytics.threatIntel.model.S3Source;
 import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
+import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
 import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
+import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import software.amazon.awssdk.core.exception.SdkException;
 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
@@ -49,10 +55,16 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
+import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp;
+
 /**
  * IOC Service implements operations that interact with retrieving IOCs from data sources,
  * parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes.
@@ -84,14 +96,14 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) {
 
     /**
      * Method takes in and calls method to rollover and bulk index a list of STIX2IOCs
+     *
      * @param saTifSourceConfig
      * @param stix2IOCList
      * @param listener
      */
     public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig,
                               List<STIX2IOC> stix2IOCList,
-                              ActionListener<STIX2IOCFetchResponse> listener)
-    {
+                              ActionListener<STIX2IOCFetchResponse> listener) {
         STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
         try {
             feedStore.indexIocs(stix2IOCList);
@@ -100,6 +112,7 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig,
             listener.onFailure(e);
         }
     }
+
     public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) {
         S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig);
         Connector<STIX2> s3Connector = constructS3Connector(s3ConnectorConfig);
@@ -144,7 +157,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL
         } catch (StsException stsException) {
             log.warn("S3Client connection test failed with StsException: ", stsException);
             listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage()));
-        } catch (SdkException sdkException ) {
+        } catch (SdkException sdkException) {
             // SdkException is a RunTimeException that doesn't have a status code.
             // Logging the full exception, and providing generic response as output.
             log.warn("S3Client connection test failed with SdkException: ", sdkException);
@@ -227,6 +240,77 @@ private String getEndpoint() {
         return "";
     }
 
+    public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) {
+        UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource();
+        switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed
+            case "csv":
+                try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) {
+                    CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl());
+                    boolean notFound = true;
+
+                    while (notFound) {
+                        CSVRecord hasHeaderRecord = reader.iterator().next();
+
+                        //if we want to skip this line and keep iterating
+                        if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') {
+                            noHeaderReader.iterator().next();
+                        } else { // we found the first line that contains information
+                            notFound = false;
+                        }
+                    }
+                    if (source.hasCsvHeader()) {
+                        parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener);
+                    } else {
+                        parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener);
+                    }
+                } catch (Exception e) {
+                    log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId());
+                    listener.onFailure(e);
+                    return;
+                }
+                break;
+            default:
+                log.error("unsupported feed format for url download:" + source.getFeedFormat());
+                listener.onFailure(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat()));
+        }
+    }
+
+    private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) throws IOException {
+        List<BulkRequest> bulkRequestList = new ArrayList<>();
+
+        UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource();
+        List<STIX2IOC> iocs = new ArrayList<>();
+        while (iterator.hasNext()) {
+            CSVRecord record = iterator.next();
+            String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null);
+            Integer colNum = source.getCsvIocValueColumnNo();
+            String iocValue = record.values()[colNum].split(" ")[0];
+            if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) {
+                log.info("Invalid IP address, skipping this ioc record: {}", iocValue);
+                continue;
+            }
+            Instant now = Instant.now();
+            STIX2IOC stix2IOC = new STIX2IOC(
+                    UUID.randomUUID().toString(),
+                    UUID.randomUUID().toString(),
+                    iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType),
+                    iocValue,
+                    "high",
+                    now,
+                    now,
+                    "",
+                    Collections.emptyList(),
+                    "",
+                    saTifSourceConfig.getId(),
+                    saTifSourceConfig.getName(),
+                    STIX2IOC.NO_VERSION
+            );
+            iocs.add(stix2IOC);
+        }
+        STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
+        feedStore.indexIocs(iocs);
+    }
+
     public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject {
         public static String IOCS_FIELD = "iocs";
         public static String TOTAL_FIELD = "total";

src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java

@@ -9,6 +9,7 @@
 import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
 import org.opensearch.securityanalytics.threatIntel.model.S3Source;
 import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
+import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,6 +56,14 @@ public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto
                     errorMsgs.add("Source must be S3_CUSTOM type");
                 }
                 break;
+            case URL_DOWNLOAD:
+                if (sourceConfigDto.getSchedule() == null) {
+                    errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type");
+                }
+                if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) {
+                    errorMsgs.add("Source must be URL_DOWNLOAD source type");
+                }
+                break;
         }
         return errorMsgs;
     }

src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java

@@ -7,11 +7,11 @@
 
 /**
  * Types of feeds threat intel can support
- * Feed types include: S3_CUSTOM
  */
 public enum SourceConfigType {
     S3_CUSTOM,
-    IOC_UPLOAD
+    IOC_UPLOAD,
+    URL_DOWNLOAD
 
 //    LICENSED,
 //

src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java

@@ -96,8 +96,4 @@ public void setIocs(List<STIX2IOCDto> iocs) {
     public String getFileName() {
         return fileName;
     }
-
-    public void setFileName(String fileName) {
-        this.fileName = fileName;
-    }
 }

src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java

@@ -20,6 +20,7 @@ public abstract class Source {
     abstract String name();
     public static final String S3_FIELD = "s3";
     public static final String IOC_UPLOAD_FIELD = "ioc_upload";
+    public static final String URL_DOWNLOAD_FIELD = "url_download";
 
     static Source readFrom(StreamInput sin) throws IOException {
         Type type = sin.readEnum(Type.class);
@@ -28,6 +29,8 @@ static Source readFrom(StreamInput sin) throws IOException {
                 return new S3Source(sin);
             case IOC_UPLOAD:
                 return new IocUploadSource(sin);
+            case URL_DOWNLOAD:
+                return new UrlDownloadSource(sin);
             default:
                 throw new IllegalStateException("Unexpected input ["+ type + "] when reading ioc store config");
         }
@@ -47,6 +50,9 @@ static Source parse(XContentParser xcp) throws IOException {
                 case IOC_UPLOAD_FIELD:
                     source = IocUploadSource.parse(xcp);
                     break;
+                case URL_DOWNLOAD_FIELD:
+                    source = UrlDownloadSource.parse(xcp);
+                    break;
             }
         }
         return source;
@@ -57,7 +63,9 @@ public void writeTo(StreamOutput out) throws IOException {}
     enum Type {
         S3(),
 
-        IOC_UPLOAD();
+        IOC_UPLOAD(),
+
+        URL_DOWNLOAD();
 
         @Override
         public String toString() {

src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java

@@ -0,0 +1,117 @@
+package org.opensearch.securityanalytics.threatIntel.model;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.net.URL;
+
+/**
+ * This is a Threat Intel Source config where the iocs are downloaded from the URL
+ */
+public class UrlDownloadSource extends Source implements Writeable, ToXContent {
+    public static final String URL_FIELD = "url";
+    public static final String FEED_FORMAT_FIELD = "feed_format";
+    public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field";
+    public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num";
+    public static final String SOURCE_NAME = "URL_DOWNLOAD";
+
+    private final URL url;
+    private final String feedFormat;
+    private final Boolean hasCsvHeader;
+    private final Integer csvIocValueColumnNo;
+
+    public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) {
+        this.url = url;
+        this.feedFormat = feedFormat;
+        this.hasCsvHeader = hasCsvHeader;
+        this.csvIocValueColumnNo = csvIocValueColumnNo;
+
+    }
+
+    public UrlDownloadSource(StreamInput sin) throws IOException {
+        this(
+                new URL(sin.readString()),
+                sin.readString(),
+                sin.readOptionalBoolean(),
+                sin.readOptionalInt()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(url.toString());
+        out.writeString(feedFormat);
+        out.writeOptionalBoolean(hasCsvHeader);
+        out.writeOptionalInt(csvIocValueColumnNo);
+    }
+
+    @Override
+    String name() {
+        return SOURCE_NAME;
+    }
+
+    public URL getUrl() {
+        return url;
+    }
+
+    public static UrlDownloadSource parse(XContentParser xcp) throws IOException {
+        URL url = null;
+        String feedFormat = null;
+        Boolean hasCsvHeader = false;
+        Integer csvIocValueColumnNo = null;
+        while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
+            String fieldName = xcp.currentName();
+            xcp.nextToken();
+            switch (fieldName) {
+                case URL_FIELD:
+                    String urlString = xcp.text();
+                    url = new URL(urlString);
+                    break;
+                case FEED_FORMAT_FIELD:
+                    feedFormat = xcp.text();
+                    break;
+                case HAS_CSV_HEADER_FIELD:
+                    hasCsvHeader = xcp.booleanValue();
+                    break;
+                case CSV_IOC_VALUE_COLUMN_NUM_FIELD:
+                    if (xcp.currentToken() == null)
+                        xcp.skipChildren();
+                    else
+                        csvIocValueColumnNo = xcp.intValue();
+                    break;
+                default:
+                    xcp.skipChildren();
+            }
+        }
+        return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject()
+                .startObject(URL_DOWNLOAD_FIELD)
+                .field(URL_FIELD, url.toString())
+                .field(FEED_FORMAT_FIELD, feedFormat)
+                .field(HAS_CSV_HEADER_FIELD, hasCsvHeader)
+                .field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo)
+                .endObject()
+                .endObject();
+    }
+
+    public String getFeedFormat() {
+        return feedFormat;
+    }
+
+    public boolean hasCsvHeader() {
+        return hasCsvHeader;
+    }
+
+    public Integer getCsvIocValueColumnNo() {
+        return csvIocValueColumnNo;
+    }
+}