diff --git a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/SlangTextualKeys.java b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/SlangTextualKeys.java index 0aacbab7b8..5c3671138a 100644 --- a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/SlangTextualKeys.java +++ b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/SlangTextualKeys.java @@ -70,6 +70,7 @@ public interface SlangTextualKeys { String PARALLEL_LOOP_KEY = "parallel_loop"; String WORKER_GROUP = "worker_group"; //&& flow String ROBOT_GROUP = "robot_group"; + String MAX_THROTTLE_KEY = "max_throttle"; //seq step String SEQ_STEP_ID_KEY = "id"; diff --git a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/ExecutableBuilder.java b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/ExecutableBuilder.java index 3a70ede8cc..7b6647e75f 100644 --- a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/ExecutableBuilder.java +++ b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/ExecutableBuilder.java @@ -58,10 +58,12 @@ import static ch.lambdaj.Lambda.filter; import static ch.lambdaj.Lambda.having; import static ch.lambdaj.Lambda.on; +import static com.google.common.collect.Maps.newHashMapWithExpectedSize; import static io.cloudslang.lang.compiler.SlangTextualKeys.DO_EXTERNAL_KEY; import static io.cloudslang.lang.compiler.SlangTextualKeys.DO_KEY; import static io.cloudslang.lang.compiler.SlangTextualKeys.FOR_KEY; import static io.cloudslang.lang.compiler.SlangTextualKeys.INPUTS_KEY; +import static io.cloudslang.lang.compiler.SlangTextualKeys.MAX_THROTTLE_KEY; import static io.cloudslang.lang.compiler.SlangTextualKeys.NAVIGATION_KEY; import static io.cloudslang.lang.compiler.SlangTextualKeys.ON_FAILURE_KEY; import static io.cloudslang.lang.compiler.SlangTextualKeys.PARALLEL_LOOP_KEY; @@ -122,7 +124,9 @@ public class ExecutableBuilder { private List stepAdditionalKeyWords = asList(LOOP_KEY, DO_KEY, DO_EXTERNAL_KEY, NAVIGATION_KEY, WORKER_GROUP, ROBOT_GROUP); - private List parallelLoopValidKeywords = asList(DO_KEY, DO_EXTERNAL_KEY, FOR_KEY, WORKER_GROUP); + private List parallelLoopValidKeywords = asList(DO_KEY, DO_EXTERNAL_KEY, FOR_KEY, + WORKER_GROUP, MAX_THROTTLE_KEY); + private List parallelLoopConstructKeywords = asList(FOR_KEY, MAX_THROTTLE_KEY); private List seqSupportedResults = asList(SUCCESS_RESULT, WARNING_RESULT, FAILURE_RESULT); @@ -184,15 +188,15 @@ public ExecutableModellingResult transformToExecutable(ParsedSlang parsedSlang, SensitivityLevel sensitivityLevel) { List errors = new ArrayList<>(); String execName = preCompileValidator.validateExecutableRawData(parsedSlang, executableRawData, errors); - String workerGroup = (String)executableRawData.get(SlangTextualKeys.WORKER_GROUP); + String workerGroup = (String) executableRawData.get(SlangTextualKeys.WORKER_GROUP); errors.addAll(preCompileValidator.checkKeyWords( - execName, - "", - executableRawData, - ListUtils.union(preExecTransformers, postExecTransformers), - ParsedSlang.Type.DECISION.equals(parsedSlang.getType()) ? - executableAdditionalKeywords : allExecutableAdditionalKeywords, - executableConstraintGroups + execName, + "", + executableRawData, + ListUtils.union(preExecTransformers, postExecTransformers), + ParsedSlang.Type.DECISION.equals(parsedSlang.getType()) ? + executableAdditionalKeywords : allExecutableAdditionalKeywords, + executableConstraintGroups ) ); @@ -292,8 +296,8 @@ public ExecutableModellingResult transformToExecutable(ParsedSlang parsedSlang, } else { preCompileValidator.validateResultsHaveNoExpression(results, execName, errors); preCompileValidator.validateResultsWithWhitelist(results, seqSupportedResults, execName, errors); - seqSteps = (List)((Map) actionRawData.get(SlangTextualKeys.SEQ_ACTION_KEY)) - .get(SEQ_STEPS_KEY); + seqSteps = (List) ((Map) actionRawData.get(SlangTextualKeys.SEQ_ACTION_KEY)) + .get(SEQ_STEPS_KEY); @SuppressWarnings("unchecked") Map settings = (Map) ((Map) actionRawData .get(SlangTextualKeys.SEQ_ACTION_KEY)) @@ -550,7 +554,13 @@ private WorkflowModellingResult compileWorkFlow(List filteredParallelLoopData = + newHashMapWithExpectedSize(parallelLoopConstructKeywords.size()); + for (String keyword : parallelLoopConstructKeywords) { + filteredParallelLoopData.put(keyword, parallelLoopRawData.remove(keyword)); + } + + parallelLoopRawData.put(PARALLEL_LOOP_KEY, filteredParallelLoopData); stepRawDataValue.putAll(parallelLoopRawData); } } @@ -676,7 +686,7 @@ private String computeWorkerGroupString(Map stepRawData) { workerGroup = (String) stepRawData.get(SlangTextualKeys.WORKER_GROUP); } else if (stepRawData.get(SlangTextualKeys.WORKER_GROUP) instanceof Map) { workerGroup = String.valueOf( - ((Map)stepRawData.get(SlangTextualKeys.WORKER_GROUP)).get(SlangTextualKeys.VALUE)); + ((Map) stepRawData.get(SlangTextualKeys.WORKER_GROUP)).get(SlangTextualKeys.VALUE)); } return workerGroup; } diff --git a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/AbstractForTransformer.java b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/AbstractForTransformer.java index 18cd66127a..12631c9eb1 100644 --- a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/AbstractForTransformer.java +++ b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/AbstractForTransformer.java @@ -9,101 +9,20 @@ *******************************************************************************/ package io.cloudslang.lang.compiler.modeller.transformers; -import io.cloudslang.lang.compiler.modeller.result.BasicTransformModellingResult; -import io.cloudslang.lang.compiler.modeller.result.TransformModellingResult; import io.cloudslang.lang.compiler.validator.ExecutableValidator; -import io.cloudslang.lang.entities.ListLoopStatement; -import io.cloudslang.lang.entities.LoopStatement; -import io.cloudslang.lang.entities.MapLoopStatement; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.commons.lang3.StringUtils; - -/** - * Date: 3/25/2015 - * - * @author Bonczidai Levente - */ public abstract class AbstractForTransformer extends AbstractInOutForTransformer { private ExecutableValidator executableValidator; // case: value in variable_name - private static final String FOR_REGEX = "^(\\s+)?(\\w+)\\s+(in)\\s+(\\w+)(\\s+)?$"; + protected static final String FOR_REGEX = "^(\\s+)?(\\w+)\\s+(in)\\s+(\\w+)(\\s+)?$"; // case: key, value - private static final String KEY_VALUE_PAIR_REGEX = "^(\\s+)?(\\w+)(\\s+)?(,)(\\s+)?(\\w+)(\\s+)?$"; - private static final String FOR_IN_KEYWORD = " in "; - - public TransformModellingResult transformToLoopStatement(String rawData, boolean isParallelLoop) { - List errors = new ArrayList<>(); - Accumulator dependencyAccumulator = extractFunctionData("${" + rawData + "}"); - if (StringUtils.isEmpty(rawData)) { - errors.add(new RuntimeException("For statement is empty.")); - return new BasicTransformModellingResult<>(null, errors); - } - - LoopStatement loopStatement = null; - String varName; - String collectionExpression; - - Pattern regexSimpleFor = Pattern.compile(FOR_REGEX); - Matcher matcherSimpleFor = regexSimpleFor.matcher(rawData); - - try { - if (matcherSimpleFor.find()) { - // case: value in variable_name - varName = matcherSimpleFor.group(2); - collectionExpression = matcherSimpleFor.group(4); - loopStatement = createLoopStatement(varName, collectionExpression, - dependencyAccumulator, isParallelLoop); - } else { - String beforeInKeyword = StringUtils.substringBefore(rawData, FOR_IN_KEYWORD); - collectionExpression = StringUtils.substringAfter(rawData, FOR_IN_KEYWORD).trim(); - - Pattern regexKeyValueFor = Pattern.compile(KEY_VALUE_PAIR_REGEX); - Matcher matcherKeyValueFor = regexKeyValueFor.matcher(beforeInKeyword); - - if (matcherKeyValueFor.find()) { - // case: key, value - String keyName = matcherKeyValueFor.group(2); - String valueName = matcherKeyValueFor.group(6); - loopStatement = createMapForLoopStatement(keyName, valueName, - collectionExpression, dependencyAccumulator); - } else { - // case: value in expression_other_than_variable_name - varName = beforeInKeyword.trim(); - loopStatement = createLoopStatement(varName, collectionExpression, - dependencyAccumulator, isParallelLoop); - } - } - } catch (RuntimeException rex) { - errors.add(rex); - } - - return new BasicTransformModellingResult<>(loopStatement, errors); - } - - private LoopStatement createMapForLoopStatement(String keyName, String valueName, - String collectionExpression, Accumulator dependencyAccumulator) { - executableValidator.validateLoopStatementVariable(keyName); - executableValidator.validateLoopStatementVariable(valueName); - return new MapLoopStatement( - keyName, - valueName, - collectionExpression, - dependencyAccumulator.getFunctionDependencies(), - dependencyAccumulator.getSystemPropertyDependencies()); - } + protected static final String KEY_VALUE_PAIR_REGEX = "^(\\s+)?(\\w+)(\\s+)?(,)(\\s+)?(\\w+)(\\s+)?$"; + protected static final String FOR_IN_KEYWORD = " in "; - private LoopStatement createLoopStatement(String varName, String collectionExpression, - Accumulator dependencyAccumulator, boolean isParallelLoop) { - executableValidator.validateLoopStatementVariable(varName); - return new ListLoopStatement(varName, collectionExpression, - dependencyAccumulator.getFunctionDependencies(), - dependencyAccumulator.getSystemPropertyDependencies(), isParallelLoop); + protected void validateLoopStatementVariable(String name) { + executableValidator.validateLoopStatementVariable(name); } public void setExecutableValidator(ExecutableValidator executableValidator) { diff --git a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ForTransformer.java b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ForTransformer.java index 8f662d1e0c..16ffed687a 100644 --- a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ForTransformer.java +++ b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ForTransformer.java @@ -11,12 +11,19 @@ import io.cloudslang.lang.compiler.CompilerConstants; import io.cloudslang.lang.compiler.SlangTextualKeys; +import io.cloudslang.lang.compiler.modeller.result.BasicTransformModellingResult; import io.cloudslang.lang.compiler.modeller.result.TransformModellingResult; +import io.cloudslang.lang.entities.ListLoopStatement; import io.cloudslang.lang.entities.LoopStatement; +import io.cloudslang.lang.entities.MapLoopStatement; import io.cloudslang.lang.entities.SensitivityLevel; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class ForTransformer extends AbstractForTransformer implements Transformer { @@ -27,7 +34,7 @@ public TransformModellingResult transform(String rawData) { @Override public TransformModellingResult transform(String rawData, SensitivityLevel sensitivityLevel) { - return transformToLoopStatement(rawData, false); + return transformToLoopStatement(rawData); } @Override @@ -40,4 +47,72 @@ public String keyToTransform() { return SlangTextualKeys.FOR_KEY; } + private TransformModellingResult transformToLoopStatement(String rawData) { + List errors = new ArrayList<>(); + Accumulator dependencyAccumulator = extractFunctionData("${" + rawData + "}"); + if (StringUtils.isEmpty(rawData)) { + errors.add(new RuntimeException("For statement is empty.")); + return new BasicTransformModellingResult<>(null, errors); + } + + LoopStatement loopStatement = null; + String varName; + String collectionExpression; + + Pattern regexSimpleFor = Pattern.compile(FOR_REGEX); + Matcher matcherSimpleFor = regexSimpleFor.matcher(rawData); + + try { + if (matcherSimpleFor.find()) { + // case: value in variable_name + varName = matcherSimpleFor.group(2); + collectionExpression = matcherSimpleFor.group(4); + loopStatement = createLoopStatement(varName, collectionExpression, dependencyAccumulator); + } else { + String beforeInKeyword = StringUtils.substringBefore(rawData, FOR_IN_KEYWORD); + collectionExpression = StringUtils.substringAfter(rawData, FOR_IN_KEYWORD).trim(); + + Pattern regexKeyValueFor = Pattern.compile(KEY_VALUE_PAIR_REGEX); + Matcher matcherKeyValueFor = regexKeyValueFor.matcher(beforeInKeyword); + + if (matcherKeyValueFor.find()) { + // case: key, value + String keyName = matcherKeyValueFor.group(2); + String valueName = matcherKeyValueFor.group(6); + loopStatement = createMapForLoopStatement(keyName, valueName, + collectionExpression, dependencyAccumulator); + } else { + // case: value in expression_other_than_variable_name + varName = beforeInKeyword.trim(); + loopStatement = createLoopStatement(varName, collectionExpression, dependencyAccumulator); + } + } + } catch (RuntimeException rex) { + errors.add(rex); + } + + return new BasicTransformModellingResult<>(loopStatement, errors); + } + + private LoopStatement createMapForLoopStatement(String keyName, String valueName, + String collectionExpression, Accumulator dependencyAccumulator) { + super.validateLoopStatementVariable(keyName); + super.validateLoopStatementVariable(valueName); + return new MapLoopStatement( + keyName, + valueName, + collectionExpression, + dependencyAccumulator.getFunctionDependencies(), + dependencyAccumulator.getSystemPropertyDependencies()); + } + + private LoopStatement createLoopStatement(String varName, String collectionExpression, + Accumulator dependencyAccumulator) { + super.validateLoopStatementVariable(varName); + return new ListLoopStatement( + varName, + collectionExpression, + dependencyAccumulator.getFunctionDependencies(), + dependencyAccumulator.getSystemPropertyDependencies()); + } } diff --git a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopForTransformer.java b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopForTransformer.java index e8a8347654..54ad6fd5b0 100644 --- a/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopForTransformer.java +++ b/cloudslang-compiler/src/main/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopForTransformer.java @@ -11,28 +11,48 @@ import io.cloudslang.lang.compiler.CompilerConstants; import io.cloudslang.lang.compiler.SlangTextualKeys; +import io.cloudslang.lang.compiler.modeller.result.BasicTransformModellingResult; import io.cloudslang.lang.compiler.modeller.result.TransformModellingResult; -import io.cloudslang.lang.entities.LoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; +import io.cloudslang.lang.entities.MapParallelLoopStatement; +import io.cloudslang.lang.entities.ParallelLoopStatement; import io.cloudslang.lang.entities.SensitivityLevel; +import io.cloudslang.lang.entities.utils.ExpressionUtils; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -/** - * Date: 3/25/2015 - * - * @author Bonczidai Levente - */ -public class ParallelLoopForTransformer extends AbstractForTransformer implements Transformer { +import static io.cloudslang.lang.compiler.SlangTextualKeys.FOR_KEY; +import static io.cloudslang.lang.compiler.SlangTextualKeys.MAX_THROTTLE_KEY; + +public class ParallelLoopForTransformer extends AbstractForTransformer + implements Transformer { @Override - public TransformModellingResult transform(String rawData) { + public TransformModellingResult transform(Object rawData) { return transform(rawData, CompilerConstants.DEFAULT_SENSITIVITY_LEVEL); } @Override - public TransformModellingResult transform(String rawData, SensitivityLevel sensitivityLevel) { - return transformToLoopStatement(rawData, true); + public TransformModellingResult transform(Object rawData, + SensitivityLevel sensitivityLevel) { + String value = null; + String throttle = null; + + if (rawData instanceof Map) { + Map parallelLoopRawData = (Map) rawData; + value = Objects.toString(parallelLoopRawData.get(FOR_KEY), null); + throttle = Objects.toString(parallelLoopRawData.get(MAX_THROTTLE_KEY), null); + } + + // throttle can be null + return transformToParallelLoopStatement(value, throttle); } @Override @@ -45,4 +65,93 @@ public String keyToTransform() { return SlangTextualKeys.PARALLEL_LOOP_KEY; } + private TransformModellingResult transformToParallelLoopStatement(String value, + String throttle) { + + List errors = new ArrayList<>(); + + if (StringUtils.isEmpty(value)) { + errors.add(new RuntimeException("For statement is empty.")); + return new BasicTransformModellingResult<>(null, errors); + } + + ParallelLoopStatement parallelLoopStatement = null; + String varName; + String collectionExpression; + Pattern regexSimpleFor = Pattern.compile(FOR_REGEX); + Matcher matcherSimpleFor = regexSimpleFor.matcher(value); + Accumulator dependencyAccumulator; + + if (throttle == null) { + dependencyAccumulator = extractFunctionData("${" + value + "}"); + } else { + dependencyAccumulator = extractFunctionData("${" + value + "}", throttle); + String throttleExpression = ExpressionUtils.extractExpression(throttle); + throttle = throttleExpression != null ? throttleExpression : throttle; + } + + try { + if (matcherSimpleFor.find()) { + // case: value in variable_name + varName = matcherSimpleFor.group(2); + collectionExpression = matcherSimpleFor.group(4); + parallelLoopStatement = createListParallelLoopStatement(varName, collectionExpression, throttle, + dependencyAccumulator); + } else { + String beforeInKeyword = StringUtils.substringBefore(value, FOR_IN_KEYWORD); + collectionExpression = StringUtils.substringAfter(value, FOR_IN_KEYWORD).trim(); + + Pattern regexKeyValueFor = Pattern.compile(KEY_VALUE_PAIR_REGEX); + Matcher matcherKeyValueFor = regexKeyValueFor.matcher(beforeInKeyword); + + if (matcherKeyValueFor.find()) { + // case: key, value + String keyName = matcherKeyValueFor.group(2); + String valueName = matcherKeyValueFor.group(6); + parallelLoopStatement = createMapParallelLoopStatement(keyName, valueName, throttle, + collectionExpression, dependencyAccumulator); + } else { + // case: value in expression_other_than_variable_name + varName = beforeInKeyword.trim(); + parallelLoopStatement = createListParallelLoopStatement(varName, collectionExpression, throttle, + dependencyAccumulator); + } + } + } catch (RuntimeException rex) { + errors.add(rex); + } + + return new BasicTransformModellingResult<>(parallelLoopStatement, errors); + } + + private ParallelLoopStatement createMapParallelLoopStatement(String keyName, + String valueName, + String throttleExpression, + String collectionExpression, + Accumulator dependencyAccumulator) { + super.validateLoopStatementVariable(keyName); + super.validateLoopStatementVariable(valueName); + return new MapParallelLoopStatement( + keyName, + valueName, + collectionExpression, + throttleExpression, + dependencyAccumulator.getFunctionDependencies(), + dependencyAccumulator.getSystemPropertyDependencies()); + } + + private ParallelLoopStatement createListParallelLoopStatement(String varName, + String collectionExpression, + String throttleExpression, + Accumulator dependencyAccumulator) { + super.validateLoopStatementVariable(varName); + return new ListParallelLoopStatement( + varName, + collectionExpression, + throttleExpression, + dependencyAccumulator.getFunctionDependencies(), + dependencyAccumulator.getSystemPropertyDependencies()); + } + + } diff --git a/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/CompileParallelLoopFlowTest.java b/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/CompileParallelLoopFlowTest.java index ed91038cf8..e3d3440abf 100644 --- a/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/CompileParallelLoopFlowTest.java +++ b/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/CompileParallelLoopFlowTest.java @@ -14,7 +14,7 @@ import io.cloudslang.lang.compiler.modeller.model.Flow; import io.cloudslang.lang.compiler.modeller.model.Step; import io.cloudslang.lang.entities.CompilationArtifact; -import io.cloudslang.lang.entities.ListLoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; import io.cloudslang.lang.entities.ResultNavigation; import io.cloudslang.lang.entities.ScoreLangConstants; import io.cloudslang.lang.entities.bindings.Output; @@ -46,11 +46,6 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -/** - * Date: 3/25/2015 - * - * @author Bonczidai Levente - */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = SlangCompilerSpringConfig.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) @@ -480,8 +475,8 @@ private void verifyNavigationValuesSuccess(Map joinBranchesActionData private void verifyParallelLoopStatement(Map addBranchesActionData) { assertTrue(addBranchesActionData.containsKey(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY)); - ListLoopStatement parallelLoopStatement = - (ListLoopStatement) addBranchesActionData.get(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY); + ListParallelLoopStatement parallelLoopStatement = + (ListParallelLoopStatement) addBranchesActionData.get(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY); assertEquals("parallel loop statement value not as expected", "value", parallelLoopStatement.getVarName()); assertEquals("parallel loop statement expression not as expected", @@ -490,7 +485,7 @@ private void verifyParallelLoopStatement(Map addBranchesActionData) { private void verifyParallelLoopStatement(Step step) { assertTrue(step.getPreStepActionData().containsKey(SlangTextualKeys.PARALLEL_LOOP_KEY)); - ListLoopStatement parallelLoopStatement = (ListLoopStatement) step.getPreStepActionData() + ListParallelLoopStatement parallelLoopStatement = (ListParallelLoopStatement) step.getPreStepActionData() .get(SlangTextualKeys.PARALLEL_LOOP_KEY); assertEquals("values", parallelLoopStatement.getExpression()); assertEquals("value", parallelLoopStatement.getVarName()); diff --git a/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopTransformerTest.java b/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopTransformerTest.java index 8ec2771f1a..ee27e16152 100644 --- a/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopTransformerTest.java +++ b/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/modeller/transformers/ParallelLoopTransformerTest.java @@ -9,12 +9,13 @@ *******************************************************************************/ package io.cloudslang.lang.compiler.modeller.transformers; +import com.google.common.collect.ImmutableMap; import io.cloudslang.lang.compiler.configuration.SlangCompilerSpringConfig; import io.cloudslang.lang.compiler.validator.ExecutableValidator; import io.cloudslang.lang.compiler.validator.ExecutableValidatorImpl; import io.cloudslang.lang.compiler.validator.SystemPropertyValidator; import io.cloudslang.lang.compiler.validator.SystemPropertyValidatorImpl; -import io.cloudslang.lang.entities.ListLoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; import io.cloudslang.lang.entities.LoopStatement; import junit.framework.Assert; import org.junit.Test; @@ -25,14 +26,10 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import static io.cloudslang.lang.compiler.SlangTextualKeys.FOR_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -/** - * Date: 4/1/2015 - * - * @author Bonczidai Levente - */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = {ParallelLoopTransformerTest.Config.class, SlangCompilerSpringConfig.class}) public class ParallelLoopTransformerTest extends TransformersTestParent { @@ -42,23 +39,24 @@ public class ParallelLoopTransformerTest extends TransformersTestParent { @Test public void testValidStatement() throws Exception { - ListLoopStatement statement = (ListLoopStatement) transformer.transform("x in collection").getTransformedData(); + ListParallelLoopStatement statement = (ListParallelLoopStatement) transformer + .transform(ImmutableMap.of(FOR_KEY, "x in collection")).getTransformedData(); Assert.assertEquals("x", statement.getVarName()); Assert.assertEquals("collection", statement.getExpression()); } @Test public void testValidStatementWithSpaces() throws Exception { - ListLoopStatement statement = - (ListLoopStatement) transformer.transform("x in range(0, 9)").getTransformedData(); + ListParallelLoopStatement statement = (ListParallelLoopStatement) transformer + .transform(ImmutableMap.of(FOR_KEY, "x in range(0, 9)")).getTransformedData(); Assert.assertEquals("x", statement.getVarName()); Assert.assertEquals("range(0, 9)", statement.getExpression()); } @Test public void testValidStatementAndTrim() throws Exception { - ListLoopStatement statement = - (ListLoopStatement) transformer.transform(" min in collection ").getTransformedData(); + ListParallelLoopStatement statement = (ListParallelLoopStatement) transformer + .transform(ImmutableMap.of(FOR_KEY, " min in collection ")).getTransformedData(); Assert.assertEquals("min", statement.getVarName()); Assert.assertEquals("collection", statement.getExpression()); } @@ -66,33 +64,34 @@ public void testValidStatementAndTrim() throws Exception { @Test public void testNoVarName() throws Exception { RuntimeException exception = assertThrows(RuntimeException.class, () -> - transformAndThrowFirstException(transformer, " in collection")); + transformAndThrowFirstException(transformer, ImmutableMap.of(FOR_KEY, " in collection"))); assertEquals("Argument[] violates character rules.", exception.getMessage()); } @Test public void testVarNameContainInvalidChars() throws Exception { RuntimeException exception = assertThrows(RuntimeException.class, () -> - transformAndThrowFirstException(transformer, "x a in collection")); + transformAndThrowFirstException(transformer, ImmutableMap.of(FOR_KEY, "x a in collection"))); assertEquals("Argument[x a] violates character rules.", exception.getMessage()); } @Test public void testNoCollectionExpression() throws Exception { RuntimeException exception = assertThrows(RuntimeException.class, () -> - transformAndThrowFirstException(transformer, "x in ")); + transformAndThrowFirstException(transformer, ImmutableMap.of(FOR_KEY, "x in "))); assertEquals("loop expression cannot be empty", exception.getMessage()); } @Test public void testMultipleInsAreTrimmed() throws Exception { - LoopStatement statement = transformer.transform(" in in in ").getTransformedData(); + LoopStatement statement = transformer + .transform(ImmutableMap.of(FOR_KEY, " in in in ")).getTransformedData(); Assert.assertEquals("in", statement.getExpression()); } @Test public void testEmptyValue() throws Exception { - LoopStatement statement = transformer.transform("").getTransformedData(); + LoopStatement statement = transformer.transform(ImmutableMap.of(FOR_KEY, "")).getTransformedData(); Assert.assertNull(statement); } diff --git a/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/scorecompiler/ExecutionStepFactoryTest.java b/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/scorecompiler/ExecutionStepFactoryTest.java index a003bc3312..44b08ab6b5 100644 --- a/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/scorecompiler/ExecutionStepFactoryTest.java +++ b/cloudslang-compiler/src/test/java/io/cloudslang/lang/compiler/scorecompiler/ExecutionStepFactoryTest.java @@ -12,6 +12,7 @@ import io.cloudslang.lang.compiler.SlangTextualKeys; import io.cloudslang.lang.entities.ExecutableType; import io.cloudslang.lang.entities.ListLoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; import io.cloudslang.lang.entities.LoopStatement; import io.cloudslang.lang.entities.ResultNavigation; import io.cloudslang.lang.entities.ScoreLangConstants; @@ -66,7 +67,7 @@ public void testCreateStartStepPutInputsUnderTheRightKey() throws Exception { @Test public void testCreateStartStepPutForUnderTheRightKey() throws Exception { LoopStatement statement = new ListLoopStatement("1", "2", new HashSet(), - new HashSet(), false); + new HashSet()); HashMap preStepData = new HashMap<>(); preStepData.put(SlangTextualKeys.FOR_KEY, statement); ExecutionStep startStep = factory.createBeginStepStep(1L, new ArrayList(), preStepData, "", "", null); @@ -215,12 +216,12 @@ public void testSplitStep() throws Exception { @Test public void testCreateAddBranchesStepPutParallelLoopUnderTheRightKey() throws Exception { - ListLoopStatement statement = new ListLoopStatement("value", "values", - new HashSet(), new HashSet(), true); + ListParallelLoopStatement statement = new ListParallelLoopStatement("value", "values", + null, new HashSet(), new HashSet()); HashMap preStepData = new HashMap<>(); preStepData.put(SlangTextualKeys.PARALLEL_LOOP_KEY, statement); ExecutionStep startStep = factory.createAddBranchesStep(2L, 5L, 3L, preStepData, "refID", "evenCoolerStep"); - ListLoopStatement actualStatement = (ListLoopStatement) startStep.getActionData() + ListParallelLoopStatement actualStatement = (ListParallelLoopStatement) startStep.getActionData() .get(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY); Assert.assertNotNull("parallel loop statement not found in action data", actualStatement); Assert.assertSame("parallel loop statement in not correctly set under the key", statement, actualStatement); diff --git a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListLoopStatement.java b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListLoopStatement.java index 24e32bca11..0c3e0a0737 100644 --- a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListLoopStatement.java +++ b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListLoopStatement.java @@ -25,20 +25,15 @@ public class ListLoopStatement extends LoopStatement implements Serializable { private static final long serialVersionUID = -540865117927676643L; - public static final String FOR_LOOP_VAR_NAME_CANNOT_BE_EMPTY = "for loop var name cannot be empty"; - public static final String PARALLEL_LOOP_VAR_NAME_CANNOT_BE_EMPTY = "parallel loop var name cannot be empty"; + private static final String FOR_LOOP_VAR_NAME_CANNOT_BE_EMPTY = "for loop var name cannot be empty"; private final String varName; public ListLoopStatement(String varName, String collectionExpression, Set functionDependencies, - Set systemPropertyDependencies, boolean isParallelLoop) { + Set systemPropertyDependencies) { super(collectionExpression, functionDependencies, systemPropertyDependencies); - String message = FOR_LOOP_VAR_NAME_CANNOT_BE_EMPTY; - if (isParallelLoop) { - message = PARALLEL_LOOP_VAR_NAME_CANNOT_BE_EMPTY; - } - Validate.notBlank(varName, message); + Validate.notBlank(varName, FOR_LOOP_VAR_NAME_CANNOT_BE_EMPTY); this.varName = varName; } diff --git a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListParallelLoopStatement.java b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListParallelLoopStatement.java new file mode 100644 index 0000000000..9a08f19941 --- /dev/null +++ b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ListParallelLoopStatement.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * (c) Copyright 2016 Hewlett-Packard Development Company, L.P. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Apache License v2.0 which accompany this distribution. + * + * The Apache License is available at + * http://www.apache.org/licenses/LICENSE-2.0 + * + *******************************************************************************/ +package io.cloudslang.lang.entities; + +import io.cloudslang.lang.entities.bindings.ScriptFunction; +import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.builder.ToStringBuilder; + +import java.io.Serializable; +import java.util.Objects; +import java.util.Set; + +public class ListParallelLoopStatement extends ParallelLoopStatement implements Serializable { + + private static final long serialVersionUID = -8939322708339882016L; + private static final String PARALLEL_LOOP_VAR_NAME_CANNOT_BE_EMPTY = "parallel loop var name cannot be empty"; + + private final String varName; + + public ListParallelLoopStatement(String varName, + String collectionExpression, + String throttleExpression, + Set functionDependencies, + Set systemPropertyDependencies) { + super(collectionExpression, throttleExpression, functionDependencies, systemPropertyDependencies); + + Validate.notBlank(varName, PARALLEL_LOOP_VAR_NAME_CANNOT_BE_EMPTY); + this.varName = varName; + } + + /** + * only here to satisfy serialization libraries + */ + @SuppressWarnings("unused") + private ListParallelLoopStatement() { + varName = null; + } + + public String getVarName() { + return varName; + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .appendSuper(super.toString()) + .append("varName", varName) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + ListParallelLoopStatement that = (ListParallelLoopStatement) o; + return Objects.equals(varName, that.varName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), varName); + } +} diff --git a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/MapParallelLoopStatement.java b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/MapParallelLoopStatement.java new file mode 100644 index 0000000000..8bd8b4a16c --- /dev/null +++ b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/MapParallelLoopStatement.java @@ -0,0 +1,86 @@ +/******************************************************************************* + * (c) Copyright 2016 Hewlett-Packard Development Company, L.P. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Apache License v2.0 which accompany this distribution. + * + * The Apache License is available at + * http://www.apache.org/licenses/LICENSE-2.0 + * + *******************************************************************************/ +package io.cloudslang.lang.entities; + +import io.cloudslang.lang.entities.bindings.ScriptFunction; +import org.apache.commons.lang3.Validate; +import org.apache.commons.lang3.builder.ToStringBuilder; + +import java.io.Serializable; +import java.util.Objects; +import java.util.Set; + +public class MapParallelLoopStatement extends ParallelLoopStatement implements Serializable { + + private static final long serialVersionUID = -2140474603664654274L; + + private final String keyName; + private final String valueName; + + public MapParallelLoopStatement(String keyName, + String valueName, + String collectionExpression, + String throttleExpression, + Set functionDependencies, + Set systemPropertyDependencies) { + super(collectionExpression, throttleExpression, functionDependencies, systemPropertyDependencies); + Validate.notBlank(keyName, "key name cannot be empty"); + Validate.notBlank(valueName, "value name cannot be empty"); + + this.keyName = keyName; + this.valueName = valueName; + } + + /** + * only here to satisfy serialization libraries + */ + @SuppressWarnings("unused") + private MapParallelLoopStatement() { + keyName = null; + valueName = null; + } + + public String getKeyName() { + return keyName; + } + + public String getValueName() { + return valueName; + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .appendSuper(super.toString()) + .append("keyName", keyName) + .append("valueName", valueName) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + MapParallelLoopStatement that = (MapParallelLoopStatement) o; + return Objects.equals(keyName, that.keyName) && Objects.equals(valueName, that.valueName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), keyName, valueName); + } +} diff --git a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ParallelLoopStatement.java b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ParallelLoopStatement.java new file mode 100644 index 0000000000..84a7173d1a --- /dev/null +++ b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/ParallelLoopStatement.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * (c) Copyright 2016 Hewlett-Packard Development Company, L.P. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Apache License v2.0 which accompany this distribution. + * + * The Apache License is available at + * http://www.apache.org/licenses/LICENSE-2.0 + * + *******************************************************************************/ +package io.cloudslang.lang.entities; + +import io.cloudslang.lang.entities.bindings.ScriptFunction; + +import java.io.Serializable; +import java.util.Objects; +import java.util.Set; + +public abstract class ParallelLoopStatement extends LoopStatement implements Serializable { + + private static final long serialVersionUID = -2601483775652385662L; + + private final String throttleExpression; + + + public ParallelLoopStatement(String expression, + String throttleExpression, + Set functionDependencies, + Set systemPropertyDependencies) { + + super(expression, functionDependencies, systemPropertyDependencies); + + this.throttleExpression = throttleExpression; + } + + /** + * only here to satisfy serialization libraries + */ + protected ParallelLoopStatement() { + throttleExpression = null; + } + + public String getThrottleExpression() { + return throttleExpression; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + ParallelLoopStatement that = (ParallelLoopStatement) o; + return Objects.equals(throttleExpression, that.throttleExpression); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), throttleExpression); + } +} diff --git a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/bindings/values/Value.java b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/bindings/values/Value.java index 99cf2fd5d2..94844eeda0 100644 --- a/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/bindings/values/Value.java +++ b/cloudslang-entities/src/main/java/io/cloudslang/lang/entities/bindings/values/Value.java @@ -32,6 +32,10 @@ public interface Value extends Serializable { String toString(); static String toStringSafe(Value value) { + if (value != null && value.isSensitive()) { + return value.get() != null ? value.get().toString() : null; + } + return value != null && value.get() != null ? value.toString() : null; } diff --git a/cloudslang-entities/src/test/java/io/cloudslang/lang/entities/DeserializeTest.java b/cloudslang-entities/src/test/java/io/cloudslang/lang/entities/DeserializeTest.java index c413266a14..7d6485e53b 100644 --- a/cloudslang-entities/src/test/java/io/cloudslang/lang/entities/DeserializeTest.java +++ b/cloudslang-entities/src/test/java/io/cloudslang/lang/entities/DeserializeTest.java @@ -98,7 +98,7 @@ public void testDeserializeResultNavigation() throws IOException { @Test public void testDeserializeListForLoopStatement() throws IOException { LoopStatement listForLoopStatement = new ListLoopStatement("varName", "expression", - new HashSet(), new HashSet(), false); + new HashSet(), new HashSet()); testToAndFromJson(listForLoopStatement, ListLoopStatement.class); } @@ -111,9 +111,9 @@ public void testDeserializeMapForLoopStatement() throws IOException { @Test public void testDeserializeParallelLoopStatement() throws IOException { - ListLoopStatement parallelLoopStatement = new ListLoopStatement("varName", "expression", - new HashSet(), new HashSet(), true); - testToAndFromJson(parallelLoopStatement, ListLoopStatement.class); + ListParallelLoopStatement parallelLoopStatement = new ListParallelLoopStatement("varName", "expression", + null, new HashSet(), new HashSet()); + testToAndFromJson(parallelLoopStatement, ListParallelLoopStatement.class); } @Test diff --git a/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/AbstractBinding.java b/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/AbstractBinding.java index 2c7e96b8eb..e380c05817 100644 --- a/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/AbstractBinding.java +++ b/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/AbstractBinding.java @@ -11,6 +11,7 @@ import io.cloudslang.lang.entities.LoopStatement; import io.cloudslang.lang.entities.MapLoopStatement; +import io.cloudslang.lang.entities.MapParallelLoopStatement; import io.cloudslang.lang.entities.bindings.prompt.Prompt; import io.cloudslang.lang.entities.bindings.values.Value; import io.cloudslang.lang.entities.bindings.values.ValueFactory; @@ -47,7 +48,7 @@ protected void validateStringValue(String errorMessagePrefix, Value value) { } protected Value getEvalResultForMap(Value evalResult, LoopStatement loopStatement, String collectionExpression) { - if (loopStatement instanceof MapLoopStatement) { + if (loopStatement instanceof MapLoopStatement || loopStatement instanceof MapParallelLoopStatement) { if (evalResult != null && evalResult.get() instanceof Map) { //noinspection unchecked Set> entrySet = diff --git a/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBinding.java b/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBinding.java index 422379eba6..e34d38f93e 100644 --- a/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBinding.java +++ b/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBinding.java @@ -9,38 +9,50 @@ *******************************************************************************/ package io.cloudslang.lang.runtime.bindings; -import io.cloudslang.lang.entities.LoopStatement; +import io.cloudslang.lang.entities.ParallelLoopStatement; import io.cloudslang.lang.entities.SystemProperty; import io.cloudslang.lang.entities.bindings.values.Value; import io.cloudslang.lang.runtime.env.Context; import org.apache.commons.collections4.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import java.util.List; import java.util.Set; +import static io.cloudslang.lang.entities.bindings.values.Value.toStringSafe; +import static java.lang.Integer.getInteger; +import static java.lang.Integer.min; +import static java.lang.Integer.parseInt; import static java.util.Objects.requireNonNull; -/** - * Date: 3/25/2015 - * - * @author Bonczidai Levente - */ @Component public class ParallelLoopBinding extends AbstractBinding { - public static final String PARALLEL_LOOP_EXPRESSION_ERROR_MESSAGE = - "Error evaluating parallel loop expression in step"; + private static final Logger logger = LogManager.getLogger(ParallelLoopBinding.class); + + private static final String EXPRESSION_ERROR_MESSAGE = "Error evaluating parallel loop expression in step"; + private static final String THROTTLE_ERROR_MESSAGE = "Error evaluating parallel loop throttle expression in step"; + private static final int DEFAULT_THROTTLE = 20; + + private final int systemMaxThrottle; + + public ParallelLoopBinding() { + int parallelThrottle = getInteger("worker.parallelMaxThrottle", DEFAULT_THROTTLE); + systemMaxThrottle = (parallelThrottle > 1 && parallelThrottle <= 1000) ? parallelThrottle : DEFAULT_THROTTLE; - public static String generateParallelLoopExpressionMessage(String nodeName, String message) { - return PARALLEL_LOOP_EXPRESSION_ERROR_MESSAGE + " '" + nodeName + "', error is: \n" + message; + logger.info("Worker parallel max throttle: " + systemMaxThrottle); } - public List bindParallelLoopList( - LoopStatement parallelLoopStatement, - Context flowContext, - Set systemProperties, - String nodeName) { + private static String generateParallelLoopExpressionMessage(String nodeName, String message) { + return EXPRESSION_ERROR_MESSAGE + " '" + nodeName + "', error is: \n" + message; + } + + public List bindParallelLoopList(ParallelLoopStatement parallelLoopStatement, + Context flowContext, + Set systemProperties, + String nodeName) { if ((parallelLoopStatement == null) || (flowContext == null) || (systemProperties == null) || (nodeName == null)) { requireNonNull(parallelLoopStatement, "parallel loop statement cannot be null"); @@ -67,4 +79,39 @@ public List bindParallelLoopList( throw new RuntimeException(generateParallelLoopExpressionMessage(nodeName, "expression is empty")); } } + + public int bindParallelLoopThrottle(ParallelLoopStatement parallelLoopStatement, + Context flowContext, + Set systemProperties, + String nodeName) { + + if (parallelLoopStatement.getThrottleExpression() == null) { + return systemMaxThrottle; + } + + if ((parallelLoopStatement == null) || (flowContext == null) || + (systemProperties == null) || (nodeName == null)) { + requireNonNull(parallelLoopStatement, "parallel loop statement cannot be null"); + requireNonNull(flowContext, "flow context cannot be null"); + requireNonNull(systemProperties, "system properties cannot be null"); + throw new NullPointerException("node name cannot be null"); + } + + try { + Value evalResult = scriptEvaluator.evalExpr(parallelLoopStatement.getThrottleExpression(), + flowContext.getImmutableViewOfVariables(), + systemProperties, + parallelLoopStatement.getFunctionDependencies()); + int throttleSize = parseInt(toStringSafe(evalResult)); + + if (throttleSize <= 0) { + throw new RuntimeException("'max_throttle' input is not valid"); + } + + return min(throttleSize, systemMaxThrottle); + } catch (Exception exc) { + throw new RuntimeException( + THROTTLE_ERROR_MESSAGE + " '" + nodeName + "', error is: \n" + exc.getMessage(), exc); + } + } } diff --git a/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/steps/ParallelLoopExecutionData.java b/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/steps/ParallelLoopExecutionData.java index 33ebf9c04a..12db218f39 100644 --- a/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/steps/ParallelLoopExecutionData.java +++ b/cloudslang-runtime/src/main/java/io/cloudslang/lang/runtime/steps/ParallelLoopExecutionData.java @@ -10,9 +10,9 @@ package io.cloudslang.lang.runtime.steps; import com.hp.oo.sdk.content.annotations.Param; -import io.cloudslang.lang.entities.ListLoopStatement; -import io.cloudslang.lang.entities.LoopStatement; -import io.cloudslang.lang.entities.MapLoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; +import io.cloudslang.lang.entities.MapParallelLoopStatement; +import io.cloudslang.lang.entities.ParallelLoopStatement; import io.cloudslang.lang.entities.ResultNavigation; import io.cloudslang.lang.entities.ScoreLangConstants; import io.cloudslang.lang.entities.bindings.Output; @@ -68,8 +68,6 @@ @Component public class ParallelLoopExecutionData extends AbstractExecutionData { - public static final String BRANCH_EXCEPTION_PREFIX = "Error running branch"; - @Autowired private ParallelLoopBinding parallelLoopBinding; @@ -78,18 +76,19 @@ public class ParallelLoopExecutionData extends AbstractExecutionData { private static final Logger logger = LogManager.getLogger(ParallelLoopExecutionData.class); - public void addBranches(@Param(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY) LoopStatement parallelLoopStatement, - @Param(RUN_ENV) RunEnvironment runEnv, - @Param(EXECUTION_RUNTIME_SERVICES) ExecutionRuntimeServices executionRuntimeServices, - @Param(ScoreLangConstants.NODE_NAME_KEY) String nodeName, + public void addBranches( + @Param(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY) ParallelLoopStatement parallelLoopStatement, + @Param(RUN_ENV) RunEnvironment runEnv, + @Param(EXECUTION_RUNTIME_SERVICES) ExecutionRuntimeServices executionRuntimeServices, + @Param(ScoreLangConstants.NODE_NAME_KEY) String nodeName, - //CHECKSTYLE:OFF: checkstyle:parametername - @Param(ExecutionParametersConsts.RUNNING_EXECUTION_PLAN_ID) Long RUNNING_EXECUTION_PLAN_ID, - //CHECKSTYLE:ON + //CHECKSTYLE:OFF: checkstyle:parametername + @Param(ExecutionParametersConsts.RUNNING_EXECUTION_PLAN_ID) Long RUNNING_EXECUTION_PLAN_ID, + //CHECKSTYLE:ON - @Param(ScoreLangConstants.NEXT_STEP_ID_KEY) Long nextStepId, - @Param(ScoreLangConstants.BRANCH_BEGIN_STEP_ID_KEY) Long branchBeginStep, - @Param(ScoreLangConstants.REF_ID) String refId) { + @Param(ScoreLangConstants.NEXT_STEP_ID_KEY) Long nextStepId, + @Param(ScoreLangConstants.BRANCH_BEGIN_STEP_ID_KEY) Long branchBeginStep, + @Param(ScoreLangConstants.REF_ID) String refId) { try { Context flowContext = runEnv.getStack().popContext(); @@ -100,9 +99,8 @@ public void addBranches(@Param(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY) L nodeName, flowContext); runEnv.putNextStepPosition(nextStepId); - // todo take from contexts: executionRuntimeServices.getThrottleSize(); - // temporary taking throttle from system properties - final Integer throttleSize = Integer.getInteger("cloudslang.worker.parallelThrottleSize", null); + + final Integer throttleSize = executionRuntimeServices.getThrottleSize(); final int splitSize = splitData.size(); final int lanesToStart = calculateNumberOfLanesToStart(splitSize, throttleSize); final List splitDataCurrentBulk = splitData.subList(0, lanesToStart); @@ -137,10 +135,11 @@ public void addBranches(@Param(ScoreLangConstants.PARALLEL_LOOP_STATEMENT_KEY) L StatefulSessionStack branchStack = branchRuntimeEnvironment.getStatefulSessionsStack(); branchStack.pushSessionsMap(new HashMap<>()); - if (parallelLoopStatement instanceof ListLoopStatement) { - branchContext.putVariable(((ListLoopStatement) parallelLoopStatement).getVarName(), splitItem); - } else if (parallelLoopStatement instanceof MapLoopStatement) { - MapLoopStatement mapLoopStatement = (MapLoopStatement) parallelLoopStatement; + if (parallelLoopStatement instanceof ListParallelLoopStatement) { + branchContext.putVariable( + ((ListParallelLoopStatement) parallelLoopStatement).getVarName(), splitItem); + } else if (parallelLoopStatement instanceof MapParallelLoopStatement) { + MapParallelLoopStatement mapLoopStatement = (MapParallelLoopStatement) parallelLoopStatement; //noinspection unchecked ImmutablePair pair = (ImmutablePair) splitItem.get(); branchContext.putVariable(mapLoopStatement.getKeyName(), pair.getLeft()); @@ -210,7 +209,7 @@ public void joinBranches(@Param(RUN_ENV) RunEnvironment runEnv, } } - private List handleFirstIteration(LoopStatement parallelLoopStatement, + private List handleFirstIteration(ParallelLoopStatement parallelLoopStatement, RunEnvironment runEnv, ExecutionRuntimeServices executionRuntimeServices, String nodeName, @@ -224,6 +223,11 @@ private List handleFirstIteration(LoopStatement parallelLoopStatement, runEnv.getSystemProperties(), nodeName); executionRuntimeServices.setSplitDataSize(splitData.size()); executionRuntimeServices.setParallelTemporaryContext(Lists.newArrayList()); + + int throttleSize = parallelLoopBinding.bindParallelLoopThrottle(parallelLoopStatement, flowContext, + runEnv.getSystemProperties(), nodeName); + executionRuntimeServices.setThrottleSize(throttleSize); + fireEvent( executionRuntimeServices, ScoreLangConstants.EVENT_SPLIT_BRANCHES, diff --git a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/LoopsBindingTest.java b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/LoopsBindingTest.java index 626c8a0a33..739343dabb 100644 --- a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/LoopsBindingTest.java +++ b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/LoopsBindingTest.java @@ -63,7 +63,7 @@ public class LoopsBindingTest { private ScriptEvaluator scriptEvaluator; private LoopStatement createBasicForStatement() { - return new ListLoopStatement("x", "[1]", new HashSet(), new HashSet(), false); + return new ListLoopStatement("x", "[1]", new HashSet(), new HashSet()); } @Test diff --git a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBindingTest.java b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBindingTest.java index 646f8629f2..41d18a7020 100644 --- a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBindingTest.java +++ b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/bindings/ParallelLoopBindingTest.java @@ -9,7 +9,7 @@ *******************************************************************************/ package io.cloudslang.lang.runtime.bindings; -import io.cloudslang.lang.entities.ListLoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; import io.cloudslang.lang.entities.SystemProperty; import io.cloudslang.lang.entities.bindings.ScriptFunction; import io.cloudslang.lang.entities.bindings.values.Value; @@ -61,8 +61,8 @@ public void passingNullParallelLoopStatementThrowsException() throws Exception { RuntimeException exception = assertThrows(RuntimeException.class, () -> parallelLoopBinding .bindParallelLoopList(null, new Context( - new HashMap(), - Collections.emptyMap()), EMPTY_SET, "nodeName")); + new HashMap(), + Collections.emptyMap()), EMPTY_SET, "nodeName")); Assert.assertEquals("parallel loop statement cannot be null", exception.getMessage()); } @@ -79,9 +79,9 @@ public void passingNullContextThrowsException() throws Exception { public void passingNullNodeNameThrowsException() throws Exception { RuntimeException exception = assertThrows(RuntimeException.class, () -> parallelLoopBinding.bindParallelLoopList( - createBasicSyncLoopStatement(), new Context( - new HashMap(), - Collections.emptyMap()), EMPTY_SET, null)); + createBasicSyncLoopStatement(), new Context( + new HashMap(), + Collections.emptyMap()), EMPTY_SET, null)); Assert.assertEquals("node name cannot be null", exception.getMessage()); } @@ -90,7 +90,7 @@ public void testParallelLoopListIsReturned() throws Exception { Map variables = new HashMap<>(); variables.put("key1", ValueFactory.create("value1")); variables.put("key2", ValueFactory.create("value2")); - final Context context = new Context(variables,Collections.emptyMap()); + final Context context = new Context(variables, Collections.emptyMap()); List expectedList = newArrayList(ValueFactory.create(1), ValueFactory.create(2), ValueFactory.create(3)); when(scriptEvaluator.evalExpr(eq("expression"), eq(variables), eq(EMPTY_SET), eq(EMPTY_FUNCTION_SET))) @@ -108,7 +108,7 @@ public void testEmptyExpressionThrowsException() throws Exception { Map variables = new HashMap<>(); variables.put("key1", ValueFactory.create("value1")); variables.put("key2", ValueFactory.create("value2")); - final Context context = new Context(variables,Collections.emptyMap()); + final Context context = new Context(variables, Collections.emptyMap()); when(scriptEvaluator.evalExpr(eq("expression"), eq(variables), eq(EMPTY_SET), eq(EMPTY_FUNCTION_SET))) .thenReturn(ValueFactory.create(newArrayList())); @@ -129,14 +129,14 @@ public void testExceptionIsPropagated() throws Exception { RuntimeException exception = assertThrows(RuntimeException.class, () -> parallelLoopBinding - .bindParallelLoopList(createBasicSyncLoopStatement(), new Context( - variables, - Collections.emptyMap()), EMPTY_SET, "nodeName")); + .bindParallelLoopList(createBasicSyncLoopStatement(), new Context( + variables, + Collections.emptyMap()), EMPTY_SET, "nodeName")); Assert.assertEquals("Error evaluating parallel loop expression in step 'nodeName', error is: \n" + "evaluation exception", exception.getMessage()); } - private ListLoopStatement createBasicSyncLoopStatement() { - return new ListLoopStatement("varName", "expression", EMPTY_FUNCTION_SET, EMPTY_PROPERTY_SET, true); + private ListParallelLoopStatement createBasicSyncLoopStatement() { + return new ListParallelLoopStatement("varName", "expression", null, EMPTY_FUNCTION_SET, EMPTY_PROPERTY_SET); } } diff --git a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/ParallelLoopStepsTest.java b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/ParallelLoopStepsTest.java index 82fd995925..8733677b92 100644 --- a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/ParallelLoopStepsTest.java +++ b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/ParallelLoopStepsTest.java @@ -13,7 +13,7 @@ import io.cloudslang.dependency.api.services.MavenConfig; import io.cloudslang.dependency.impl.services.DependencyServiceImpl; import io.cloudslang.dependency.impl.services.MavenConfigImpl; -import io.cloudslang.lang.entities.ListLoopStatement; +import io.cloudslang.lang.entities.ListParallelLoopStatement; import io.cloudslang.lang.entities.ResultNavigation; import io.cloudslang.lang.entities.ScoreLangConstants; import io.cloudslang.lang.entities.bindings.Output; @@ -69,11 +69,6 @@ import static org.mockito.Mockito.when; import static org.python.google.common.collect.Lists.newArrayList; -/** - * Date: 4/7/2015 - * - * @author Bonczidai Levente - */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = ParallelLoopStepsTest.Config.class) public class ParallelLoopStepsTest { @@ -103,8 +98,8 @@ public void resetMocks() { @Test public void testBranchesAreCreated() throws Exception { // prepare arguments - ListLoopStatement parallelLoopStatement = new ListLoopStatement("varName", "expression", - new HashSet(), new HashSet(), true); + ListParallelLoopStatement parallelLoopStatement = new ListParallelLoopStatement("varName", "expression", null, + new HashSet(), new HashSet()); RunEnvironment runEnvironment = new RunEnvironment(); Map variables = new HashMap<>(); @@ -121,6 +116,7 @@ public void testBranchesAreCreated() throws Exception { when(parallelLoopBinding.bindParallelLoopList(eq(parallelLoopStatement), eq(context), eq(runEnvironment.getSystemProperties()), eq(nodeName))) .thenReturn(expectedSplitData); + when(executionRuntimeServices.getThrottleSize()).thenReturn(3); Long branchBeginStepId = 3L; // call method @@ -159,8 +155,8 @@ public void testBranchesAreCreated() throws Exception { @Test public void testAddBranchesEventsAreFired() throws Exception { // prepare arguments - ListLoopStatement parallelLoopStatement = new ListLoopStatement("varName", "expression", - new HashSet(), new HashSet(), true); + ListParallelLoopStatement parallelLoopStatement = new ListParallelLoopStatement("varName", "expression", null, + new HashSet(), new HashSet()); RunEnvironment runEnvironment = new RunEnvironment(); Map variables = new HashMap<>(); @@ -177,6 +173,7 @@ public void testAddBranchesEventsAreFired() throws Exception { when(parallelLoopBinding.bindParallelLoopList(eq(parallelLoopStatement), eq(context), eq(runEnvironment.getSystemProperties()), eq(nodeName))) .thenReturn(expectedSplitData); + when(executionRuntimeServices.getThrottleSize()).thenReturn(3); Long branchBeginStepId = 0L; // call method diff --git a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/StepExecutionDataTest.java b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/StepExecutionDataTest.java index 5b633b0137..4a0314227d 100644 --- a/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/StepExecutionDataTest.java +++ b/cloudslang-runtime/src/test/java/io/cloudslang/lang/runtime/steps/StepExecutionDataTest.java @@ -110,7 +110,7 @@ private RunEnvironment createRunEnvironment() { private LoopStatement createBasicForStatement(String varName, String collectionExpression) { return new ListLoopStatement(varName, collectionExpression, - new HashSet(), new HashSet(), false); + new HashSet(), new HashSet()); } @Before