Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Force using snakeyaml version 1.32 to fix CVE issue #535

Merged
merged 17 commits into from
Oct 4, 2022
Merged
12 changes: 6 additions & 6 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ buildscript {
notification_version = System.getProperty("notification.version", opensearch_build)
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build)
kotlin_version = System.getProperty("kotlin.version", "1.4.0")
kotlin_version = System.getProperty("kotlin.version", "1.6.10")

opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
job_scheduler_no_snapshot = job_scheduler_version.replace("-SNAPSHOT","")
Expand All @@ -39,8 +39,8 @@ buildscript {
classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.17.1"
classpath "org.jacoco:org.jacoco.agent:0.8.5"
classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.21.0"
classpath "org.jacoco:org.jacoco.agent:0.8.7"
}
}

Expand Down Expand Up @@ -73,8 +73,8 @@ configurations.all {
force 'org.apache.httpcomponents:httpclient-osgi:4.5.13'
force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
force 'com.fasterxml.jackson.core:jackson-databind:2.10.4'
force 'org.yaml:snakeyaml:1.26'
force 'com.fasterxml.jackson.core:jackson-databind:2.13.4'
force 'org.yaml:snakeyaml:1.32'
force 'org.codehaus.plexus:plexus-utils:3.0.24'
}
}
Expand Down Expand Up @@ -602,4 +602,4 @@ task updateVersion {
// String tokenization to support -SNAPSHOT
ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
}
}
}
2 changes: 1 addition & 1 deletion detekt.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# TODO: Remove this before initial release, only for developmental purposes
build:
maxIssues: 10
maxIssues: 20

exceptions:
TooGenericExceptionCaught:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ abstract class Step(val name: String, val isSafeToDisableOn: Boolean = true) {

companion object {
fun read(streamInput: StreamInput): StepStatus {
return valueOf(streamInput.readString().toUpperCase(Locale.ROOT))
return valueOf(streamInput.readString().uppercase(Locale.ROOT))
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ data class ActionRetry(

when (fieldName) {
COUNT_FIELD -> count = xcp.longValue()
BACKOFF_FIELD -> backoff = Backoff.valueOf(xcp.text().toUpperCase(Locale.ROOT))
BACKOFF_FIELD -> backoff = Backoff.valueOf(xcp.text().uppercase(Locale.ROOT))
DELAY_FIELD -> delay = TimeValue.parseTimeValue(xcp.text(), DELAY_FIELD)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ data class StepMetaData(
when (fieldName) {
NAME -> name = xcp.text()
START_TIME -> startTime = xcp.longValue()
STEP_STATUS -> stepStatus = Step.StepStatus.valueOf(xcp.text().toUpperCase(Locale.ROOT))
STEP_STATUS -> stepStatus = Step.StepStatus.valueOf(xcp.text().uppercase(Locale.ROOT))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
*/

package org.opensearch.indexmanagement

import org.apache.logging.log4j.LogManager
import org.opensearch.action.ActionRequest
import org.opensearch.action.ActionResponse
Expand Down Expand Up @@ -38,6 +37,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinat
import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner
import org.opensearch.indexmanagement.indexstatemanagement.MetadataService
import org.opensearch.indexmanagement.indexstatemanagement.SkipExecution
import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig
import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestAddPolicyAction
Expand Down Expand Up @@ -73,7 +73,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retr
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
import org.opensearch.indexmanagement.migration.ISMTemplateService
import org.opensearch.indexmanagement.refreshanalyzer.RefreshSearchAnalyzerAction
import org.opensearch.indexmanagement.refreshanalyzer.RestRefreshSearchAnalyzerAction
import org.opensearch.indexmanagement.refreshanalyzer.TransportRefreshSearchAnalyzerAction
Expand Down Expand Up @@ -251,15 +250,15 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
indexManagementExtensions.forEach { extension ->
val extensionName = extension.getExtensionName()
if (extensionName in extensions) {
throw IllegalStateException("Multiple extensions of IndexManagement have same name $extensionName - not supported")
error("Multiple extensions of IndexManagement have same name $extensionName - not supported")
}
extension.getISMActionParsers().forEach { parser ->
ISMActionsParser.instance.addParser(parser, extensionName)
}
indexMetadataServices.add(extension.getIndexMetadataService())
extension.overrideClusterStateIndexUuidSetting()?.let {
if (customIndexUUIDSetting != null) {
throw IllegalStateException(
error(
"Multiple extensions of IndexManagement plugin overriding ClusterStateIndexUUIDSetting - not supported"
)
}
Expand Down Expand Up @@ -359,7 +358,9 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
)

indexMetadataProvider = IndexMetadataProvider(
settings, client, clusterService,
settings,
client,
clusterService,
hashMapOf(
DEFAULT_INDEX_TYPE to DefaultIndexMetadataService(customIndexUUIDSetting)
)
Expand All @@ -386,7 +387,13 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin

val managedIndexCoordinator = ManagedIndexCoordinator(
environment.settings(),
client, clusterService, threadPool, indexManagementIndices, metadataService, templateService, indexMetadataProvider
client,
clusterService,
threadPool,
indexManagementIndices,
metadataService,
templateService,
indexMetadataProvider
)

return listOf(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import org.opensearch.index.query.QueryBuilders
import org.opensearch.indexmanagement.IndexManagementIndices
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig
import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig
Expand All @@ -73,7 +74,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed
import org.opensearch.indexmanagement.indexstatemanagement.util.isPolicyCompleted
import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest
import org.opensearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest
import org.opensearch.indexmanagement.migration.ISMTemplateService
import org.opensearch.indexmanagement.opensearchapi.IndexManagementSecurityContext
import org.opensearch.indexmanagement.opensearchapi.contentParser
import org.opensearch.indexmanagement.opensearchapi.parseFromSearchResponse
Expand Down Expand Up @@ -128,17 +128,26 @@ class ManagedIndexCoordinator(
private var scheduledTemplateMigration: Scheduler.Cancellable? = null

@Volatile private var lastFullSweepTimeNano = System.nanoTime()

@Volatile private var indexStateManagementEnabled = INDEX_STATE_MANAGEMENT_ENABLED.get(settings)

@Volatile private var metadataServiceEnabled = METADATA_SERVICE_ENABLED.get(settings)

@Volatile private var sweepPeriod = SWEEP_PERIOD.get(settings)

@Volatile private var retryPolicy =
BackoffPolicy.constantBackoff(COORDINATOR_BACKOFF_MILLIS.get(settings), COORDINATOR_BACKOFF_COUNT.get(settings))

@Volatile private var templateMigrationEnabled: Boolean = true

@Volatile private var templateMigrationEnabledSetting = TEMPLATE_MIGRATION_CONTROL.get(settings)

@Volatile private var jobInterval = JOB_INTERVAL.get(settings)

@Volatile private var jobJitter = JITTER.get(settings)

@Volatile private var isMaster = false

@Volatile private var onMasterTimeStamp: Long = 0L

init {
Expand Down Expand Up @@ -168,8 +177,7 @@ class ManagedIndexCoordinator(
if (!templateMigrationEnabled) scheduledTemplateMigration?.cancel()
else initTemplateMigration(it)
}
clusterService.clusterSettings.addSettingsUpdateConsumer(COORDINATOR_BACKOFF_MILLIS, COORDINATOR_BACKOFF_COUNT) {
millis, count ->
clusterService.clusterSettings.addSettingsUpdateConsumer(COORDINATOR_BACKOFF_MILLIS, COORDINATOR_BACKOFF_COUNT) { millis, count ->
retryPolicy = BackoffPolicy.constantBackoff(millis, count)
}
}
Expand Down Expand Up @@ -556,14 +564,15 @@ class ManagedIndexCoordinator(

logger.info("Performing ISM template migration.")
if (enableSetting == 0L) {
if (onMasterTimeStamp != 0L)
if (onMasterTimeStamp != 0L) {
templateService.doMigration(Instant.ofEpochMilli(onMasterTimeStamp))
else {
} else {
logger.error("No valid onMaster time cached, cancel ISM template migration job.")
scheduledTemplateMigration?.cancel()
}
} else
} else {
templateService.doMigration(Instant.ofEpochMilli(enableSetting))
}
} catch (e: Exception) {
logger.error("Failed to migrate ISM template", e)
}
Expand Down Expand Up @@ -596,7 +605,8 @@ class ManagedIndexCoordinator(

// Get the matching policyIds for applicable indices
val updateMatchingIndicesReqs = createManagedIndexRequests(
clusterService.state(), unManagedIndices.map { (indexName, _) -> indexName }
clusterService.state(),
unManagedIndices.map { (indexName, _) -> indexName }
)

// check all managed indices, if the index has already been deleted
Expand Down Expand Up @@ -701,7 +711,10 @@ class ManagedIndexCoordinator(
mRes.forEach {
if (it.response.isExists) {
result[it.id] = contentParser(it.response.sourceAsBytesRef).parseWithType(
it.response.id, it.response.seqNo, it.response.primaryTerm, ManagedIndexConfig.Companion::parse
it.response.id,
it.response.seqNo,
it.response.primaryTerm,
ManagedIndexConfig.Companion::parse
)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.indexmanagement.migration
package org.opensearch.indexmanagement.indexstatemanagement.migration

import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
Expand Down Expand Up @@ -254,7 +254,7 @@ class ISMTemplateService(

private fun populateV2ISMTemplateMap(policyID: String, indexPatterns: List<String>, priority: Int) {
var v1Increment = 0
val v1MaxOrder = v1orderToBucketIncrement.keys.max()
val v1MaxOrder = v1orderToBucketIncrement.keys.maxOrNull()
if (v1MaxOrder != null) {
v1Increment = v1MaxOrder + v1orderToBucketIncrement.values.sum()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken
import org.opensearch.common.xcontent.XContentType
import org.opensearch.indexmanagement.opensearchapi.string
import java.io.IOException
import java.lang.IllegalStateException

/**
* A value object that represents a Chime message. Chime message will be
Expand Down Expand Up @@ -61,7 +60,7 @@ data class Chime(val url: String) : ToXContent, Writeable {
when (fieldName) {
URL -> url = xcp.text()
else -> {
throw IllegalStateException("Unexpected field: $fieldName, while parsing Chime destination")
error("Unexpected field: $fieldName, while parsing Chime destination")
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import org.opensearch.common.xcontent.XContentParser
import org.opensearch.common.xcontent.XContentParser.Token
import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken
import java.io.IOException
import java.lang.IllegalStateException

/**
* A value object that represents a Custom webhook message. Webhook message will be
Expand Down Expand Up @@ -121,7 +120,7 @@ data class CustomWebhook(
USERNAME_FIELD -> username = xcp.textOrNull()
PASSWORD_FIELD -> password = xcp.textOrNull()
else -> {
throw IllegalStateException("Unexpected field: $fieldName, while parsing custom webhook destination")
error("Unexpected field: $fieldName, while parsing custom webhook destination")
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken
import org.opensearch.common.xcontent.XContentType
import org.opensearch.indexmanagement.opensearchapi.string
import java.io.IOException
import java.lang.IllegalStateException

/**
* A value object that represents a Slack message. Slack message will be
Expand Down Expand Up @@ -61,7 +60,7 @@ data class Slack(val url: String) : ToXContent, Writeable {
when (fieldName) {
URL -> url = xcp.text()
else -> {
throw IllegalStateException("Unexpected field: $fieldName, while parsing Slack destination")
error("Unexpected field: $fieldName, while parsing Slack destination")
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ class RollupIndexer(
is InternalMin -> aggResults[it.name] = it.value
is InternalValueCount -> aggResults[it.name] = it.value
is InternalAvg -> aggResults[it.name] = it.value
else -> throw IllegalStateException("Found aggregation in composite result that is not supported [${it.type} - ${it.name}]")
else -> error("Found aggregation in composite result that is not supported [${it.type} - ${it.name}]")
}
}
mapOfKeyValues.putAll(aggResults)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ object RollupRunner :
) {
client.suspendUntil { listener: ActionListener<GetRollupResponse> ->
execute(GetRollupAction.INSTANCE, GetRollupRequest(updatableJob.id, null, "_local"), listener)
}.rollup ?: throw IllegalStateException("Unable to get rollup job")
}.rollup ?: error("Unable to get rollup job")
}
}
is RollupResult.Failure -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ inline fun <reified T> Rollup.findMatchingMetricField(field: String): String {
}
}
}
throw IllegalStateException("Did not find matching rollup metric")
error("Did not find matching rollup metric")
}

@Suppress("NestedBlockDepth", "ComplexMethod")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import org.opensearch.indexmanagement.transform.model.TransformValidationResult
import org.opensearch.indexmanagement.transform.settings.TransformSettings
import org.opensearch.monitor.jvm.JvmService
import org.opensearch.transport.RemoteTransportException
import java.lang.IllegalStateException

@Suppress("SpreadOperator", "ReturnCount", "ThrowsCount")
class TransformValidator(
Expand Down Expand Up @@ -93,7 +92,7 @@ class TransformValidator(
private suspend fun validateIndex(index: String, transform: Transform): List<String> {
val request = GetMappingsRequest().indices(index)
val result: GetMappingsResponse =
client.admin().indices().suspendUntil { getMappings(request, it) } ?: throw IllegalStateException(
client.admin().indices().suspendUntil { getMappings(request, it) } ?: error(
"GetMappingResponse for [$index] was null"
)
return validateMappingsResponse(index, result, transform)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ data class TransformMetadata(
TRANSFORM_ID_FIELD -> transformId = xcp.text()
AFTER_KEY_FIELD -> afterkey = xcp.map()
LAST_UPDATED_AT_FIELD -> lastUpdatedAt = xcp.instant()
STATUS_FIELD -> status = Status.valueOf(xcp.text().toUpperCase(Locale.ROOT))
STATUS_FIELD -> status = Status.valueOf(xcp.text().uppercase(Locale.ROOT))
FAILURE_REASON -> failureReason = xcp.textOrNull()
STATS_FIELD -> stats = TransformStats.parse(xcp)
SHARD_ID_TO_GLOBAL_CHECKPOINT_FIELD ->
Expand Down
Loading