Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adjusted IOCTypes usage. #1156

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified security-analytics-commons-1.0.0.jar
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
xcp.nextToken();

switch (fieldName) {
// synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
case STIX2.ID_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
id = xcp.text();
Expand All @@ -90,7 +89,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
name = xcp.text();
break;
case STIX2.TYPE_FIELD:
type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
Expand Down Expand Up @@ -177,7 +176,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder.startObject()
.field(STIX2IOC.ID_FIELD, ioc.getId())
.field(STIX2IOC.NAME_FIELD, ioc.getName())
.field(STIX2IOC.TYPE_FIELD, ioc.getType())
.field(STIX2IOC.TYPE_FIELD, ioc.getType().toString())
.field(STIX2IOC.VALUE_FIELD, ioc.getValue())
.field(STIX2IOC.SEVERITY_FIELD, ioc.getSeverity())
.timeField(STIX2IOC.CREATED_FIELD, ioc.getCreated())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
this(
sin.readString(), // id
sin.readString(), // name
sin.readEnum(IOCType.class), // type
new IOCType(sin.readString()), // type
sin.readString(), // value
sin.readString(), // severity
sin.readInstant(), // created
Expand Down Expand Up @@ -142,7 +142,7 @@ public static STIX2IOC readFrom(StreamInput sin) throws IOException {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(super.getId());
out.writeString(super.getName());
out.writeEnum(super.getType());
out.writeString(super.getType().toString());
out.writeString(super.getValue());
out.writeString(super.getSeverity());
out.writeInstant(super.getCreated());
Expand All @@ -160,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(ID_FIELD, super.getId())
.field(NAME_FIELD, super.getName())
.field(TYPE_FIELD, super.getType())
.field(TYPE_FIELD, super.getType().toString())
.field(VALUE_FIELD, super.getValue())
.field(SEVERITY_FIELD, super.getSeverity());
XContentUtils.buildInstantAsField(builder, super.getCreated(), CREATED_FIELD);
Expand Down Expand Up @@ -205,7 +205,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
name = xcp.text();
break;
case TYPE_FIELD:
type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
type = new IOCType(xcp.text());
break;
case VALUE_FIELD:
value = xcp.text();
Expand Down Expand Up @@ -292,8 +292,8 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
public void validate() throws IllegalArgumentException {
if (super.getType() == null) {
throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
} else if (!Arrays.asList(IOCType.values()).contains(super.getType())) {
logger.debug("Unsupported IOCType: {}", super.getType());
} else if (!IOCType.supportedType(super.getType().toString())) {
logger.debug("Unsupported IOCType: {}", super.getType().toString());
throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public static STIX2IOCDto readFrom(StreamInput sin) throws IOException {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeString(name);
out.writeEnum(type);
out.writeString(type.toString());
out.writeString(value);
out.writeString(severity);
out.writeInstant(created);
Expand All @@ -120,7 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder.startObject()
.field(STIX2IOC.ID_FIELD, id)
.field(STIX2IOC.NAME_FIELD, name)
.field(STIX2IOC.TYPE_FIELD, type)
.field(STIX2IOC.TYPE_FIELD, type.toString())
.field(STIX2IOC.VALUE_FIELD, value)
.field(STIX2IOC.SEVERITY_FIELD, severity)
.timeField(STIX2IOC.CREATED_FIELD, created)
Expand Down Expand Up @@ -161,7 +161,6 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
xcp.nextToken();

switch (fieldName) {
// synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
case STIX2.ID_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
id = xcp.text();
Expand All @@ -176,7 +175,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
name = xcp.text();
break;
case STIX2.TYPE_FIELD:
type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
type = new IOCType(xcp.text());
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
package org.opensearch.securityanalytics.services;

import com.amazonaws.services.s3.AmazonS3;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.securityanalytics.commons.connector.Connector;
import org.opensearch.securityanalytics.commons.connector.S3Connector;
import org.opensearch.securityanalytics.commons.connector.codec.InputCodec;
Expand All @@ -21,6 +23,7 @@
import java.util.List;

public class STIX2IOCConnectorFactory extends UnaryParameterCachingFactory<FeedConfiguration, Connector<STIX2>> {
private static final Logger logger = LogManager.getLogger(STIX2IOCConnectorFactory.class);
private final InputCodecFactory inputCodecFactory;
private final S3ClientFactory s3ClientFactory;

Expand All @@ -31,7 +34,7 @@ public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final

protected Connector<STIX2> doCreate(FeedConfiguration feedConfiguration) {
final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration);
// TODO hurneyt add debug log for which location gets used
logger.debug("FeedLocation: {}", feedLocation);
switch(feedLocation) {
case S3: return createS3Connector(feedConfiguration);
default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,10 @@ public void accept(final STIX2 ioc) {
feedStore.getSaTifSourceConfig().getName()
);

// TODO hurneyt refactor once the enum values are updated
// If the IOC received is not a type listed for the config, do not add it to the queue
if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().name())) {
if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().name(), feedStore.getSaTifSourceConfig().getId(),
stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(),
stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
);
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,13 @@ public class STIX2IOCFeedStore implements FeedStore {
public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER;

private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class);

Instant startTime = Instant.now();

private Client client;
private ClusterService clusterService;
private SATIFSourceConfig saTifSourceConfig;

// TODO hurneyt FetchIocsActionResponse is just a placeholder response type for now
private ActionListener<STIX2IOCFetchService.STIX2IOCFetchResponse> baseListener;

// TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
private Integer batchSize;

public STIX2IOCFeedStore(
Expand Down Expand Up @@ -97,7 +94,6 @@ public void storeIOCs(Map<IOC, UpdateAction> actionToIOCs) {
for (Map.Entry<UpdateAction, List<STIX2IOC>> entry : iocsSortedByAction.entrySet()) {
switch (entry.getKey()) {
case DELETE:
// TODO hurneyt consider whether DELETE actions should be handled elsewhere
break;
case UPSERT:
try {
Expand All @@ -119,7 +115,7 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
initFeedIndex(newActiveIndex, ActionListener.wrap(
r -> {
saTifSourceConfig.getIocTypes().forEach(type -> {
IOCType iocType = IOCType.fromString(type);
IOCType iocType = new IOCType(type);
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ public class STIX2IOCFetchService {
private STIX2IOCConnectorFactory connectorFactory;
private S3ClientFactory s3ClientFactory;

// TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
private Integer batchSize;
private String internalAuthEndpoint = "";

Expand Down Expand Up @@ -126,7 +125,6 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList
listener.onFailure(e);
}

// TODO consider passing listener into the flush IOC function
try {
consumer.flushIOCs();
} catch (Exception e) {
Expand Down Expand Up @@ -285,15 +283,15 @@ private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SA
String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null);
Integer colNum = source.getCsvIocValueColumnNo();
String iocValue = record.values()[colNum].split(" ")[0];
if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) {
if (iocType.equalsIgnoreCase(IOCType.IPV4_TYPE) && !isValidIp(iocValue)) {
log.info("Invalid IP address, skipping this ioc record: {}", iocValue);
continue;
}
Instant now = Instant.now();
STIX2IOC stix2IOC = new STIX2IOC(
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType),
iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType),
iocValue,
"high",
now,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.securityanalytics.threatIntel.common;

import org.opensearch.securityanalytics.commons.model.IOC;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
Expand All @@ -22,15 +23,12 @@
public class SourceConfigDtoValidator {
public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) {
List<String> errorMsgs = new ArrayList<>();
List<String> iocTypeEnumNames = Arrays.stream(IOCType.values())
.map(Enum::name)
.collect(Collectors.toList());

if (sourceConfigDto.getIocTypes().isEmpty()) {
errorMsgs.add("Must specify at least one IOC type");
} else {
for (String s: sourceConfigDto.getIocTypes()) {
if (false == iocTypeEnumNames.contains(s)) {
if (!IOCType.supportedType(s)) {
errorMsgs.add("Invalid IOC type: " + s);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
Expand Down Expand Up @@ -375,9 +376,9 @@ private static SearchRequest getSearchRequestForIocType(List<String> indices, St
SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0]));
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// add the iocs sublist
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist));
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD + ".keyword", iocsSublist));
// add ioc type filter
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase()));
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD + ".keyword", iocType.toLowerCase(Locale.ROOT)));
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For context, after changing the IOCTypes from using underscores (e.g., domain_name) to using hyphens (e.g., domain-name), it was necessary to append the fields with .keyword in order for this search request to return results. This change is needed to ensure the potentially-hyphenated values of the fields are being queried as-is.

Without these changes, the query returns 0 results despite a match_all query confirming there are docs in the concrete index. This was the behavior regardless of whether the index mappings were specified during index creation.

searchRequest.source().query(boolQueryBuilder);
return searchRequest;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,21 +101,23 @@ public IocToIndexDetails(IOCType iocType, String indexPattern, String activeInde
}

public IocToIndexDetails(StreamInput sin) throws IOException {
this(sin.readEnum(IOCType.class),
this(
new IOCType(sin.readString()),
sin.readString(),
sin.readString());
sin.readString()
);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(iocType);
out.writeString(iocType.toString());
out.writeString(indexPattern);
out.writeString(activeIndex);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.field(IOC_TYPE_FIELD, iocType)
.field(IOC_TYPE_FIELD, iocType.toString())
.field(INDEX_PATTERN_FIELD, indexPattern)
.field(ACTIVE_INDEX_FIELD, activeIndex)
.endObject();
Expand Down Expand Up @@ -150,7 +152,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {

public static IOCType toIocType(String name) {
try {
return IOCType.fromString(name);
return new IOCType(name);
} catch (IllegalArgumentException e) {
log.error("Invalid Ioc type, cannot be parsed.", e);
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public void onFailure(Exception e) {
Instant now = Instant.now();
String iocType = null;
if (tifMetadata.getIocType().equalsIgnoreCase("ip")) {
iocType = IOCType.ipv4_addr.toString();
iocType = IOCType.IPV4_TYPE;
}
satifSourceConfigManagementService.createOrUpdateTifSourceConfig(
new SATIFSourceConfigDto(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.opensearch.rest.RestRequest;
import org.opensearch.search.SearchHit;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.model.STIX2IOCDto;
import org.opensearch.securityanalytics.services.STIX2IOCFetchService;
Expand Down Expand Up @@ -200,12 +201,12 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
List<STIX2IOC> validStix2IocList = new ArrayList<>();
// If the IOC received is not a type listed for the config, do not add it to the queue
for (STIX2IOC stix2IOC : stix2IOCList) {
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().name())) {
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
validStix2IocList.add(stix2IOC);
} else {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().name(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
stix2IOC.getType().toString(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
);
}
}
Expand Down Expand Up @@ -354,7 +355,7 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
Set<String> concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse);

// remove ioc types not specified in list
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name()));
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));

// get the active indices
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex()));
Expand Down Expand Up @@ -467,7 +468,7 @@ private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> l
if (newIocStoreConfig instanceof DefaultIocStoreConfig) {
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig;
// remove ioc types not specified in list
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name()));
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));
updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig);
}
// Update source config as succeeded, change state back to available
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2890,7 +2890,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig(
schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS);
}
if (iocStoreConfig == null) {
iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.domain_name, "indexPattern", "writeIndex")));
iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex")));
}
if (iocTypes == null) {
iocTypes = List.of("ip");
Expand Down
Loading
Loading