From 285008ff79a833a27b7c44fc9db87abdde987dc8 Mon Sep 17 00:00:00 2001 From: Denes Bodo Date: Wed, 17 Jul 2024 19:03:50 +0200 Subject: [PATCH] OOZIE-3719: Improve coordinator scope range checking --- .../org/apache/oozie/CoordinatorEngine.java | 45 +----- .../main/java/org/apache/oozie/ErrorCode.java | 1 + .../org/apache/oozie/coord/CoordUtils.java | 153 +++++++++++++----- .../apache/oozie/servlet/V1JobServlet.java | 62 ++++++- .../apache/oozie/servlet/V2JobServlet.java | 6 +- core/src/main/resources/oozie-default.xml | 11 ++ .../oozie/coord/TestCoordUtilsNoServices.java | 152 +++++++++++++++++ .../oozie/servlet/DagServletTestCase.java | 17 +- .../oozie/servlet/TestV1JobServlet.java | 129 +++++++++++++++ .../oozie/servlet/TestV2JobServlet.java | 37 +++++ 10 files changed, 521 insertions(+), 92 deletions(-) create mode 100644 core/src/test/java/org/apache/oozie/coord/TestCoordUtilsNoServices.java diff --git a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java index 07bab122e5..4506ed7877 100644 --- a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java +++ b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,6 +61,7 @@ import org.apache.oozie.command.coord.CoordSuspendXCommand; import org.apache.oozie.command.coord.CoordUpdateXCommand; import org.apache.oozie.command.coord.CoordWfActionInfoXCommand; +import org.apache.oozie.coord.CoordUtils; import org.apache.oozie.dependency.ActionDependency; import org.apache.oozie.executor.jpa.CoordActionQueryExecutor; import org.apache.oozie.executor.jpa.CoordJobQueryExecutor; @@ -314,48 +314,7 @@ public void streamLog(String jobId, String logRetrievalScope, String logRetrieva // if coordinator action logs are to be retrieved based on action id range if (logRetrievalType.equals(RestConstants.JOB_LOG_ACTION)) { // Use set implementation that maintains order or elements to achieve reproducibility: - Set actionSet = new LinkedHashSet(); - String[] list = logRetrievalScope.split(","); - for (String s : list) { - s = s.trim(); - if (s.contains("-")) { - String[] range = s.split("-"); - if (range.length != 2) { - throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s - + "'"); - } - int start; - int end; - try { - start = Integer.parseInt(range[0].trim()); - } catch (NumberFormatException ne) { - throw new CommandException(ErrorCode.E0302, "could not parse " + range[0].trim() + "into an integer", - ne); - } - try { - end = Integer.parseInt(range[1].trim()); - } catch (NumberFormatException ne) { - throw new CommandException(ErrorCode.E0302, "could not parse " + range[1].trim() + "into an integer", - ne); - } - if (start > end) { - throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s + "'"); - } - for (int i = start; i <= end; i++) { - actionSet.add(jobId + "@" + i); - } - } - else { - try { - Integer.parseInt(s); - } - catch (NumberFormatException ne) { - throw new CommandException(ErrorCode.E0302, "format is wrong for action id'" + s - + "'. Integer only."); - } - actionSet.add(jobId + "@" + s); - } - } + final Set actionSet = CoordUtils.getActionsIds(jobId, logRetrievalScope); if (actionSet.size() >= maxNumActionsForLog) { throw new CommandException(ErrorCode.E0302, diff --git a/core/src/main/java/org/apache/oozie/ErrorCode.java b/core/src/main/java/org/apache/oozie/ErrorCode.java index a8053004e5..beae893a83 100644 --- a/core/src/main/java/org/apache/oozie/ErrorCode.java +++ b/core/src/main/java/org/apache/oozie/ErrorCode.java @@ -66,6 +66,7 @@ public enum ErrorCode { E0306(XLog.STD, "Invalid parameter"), E0307(XLog.STD, "Runtime error [{0}]"), E0308(XLog.STD, "Could not parse date range parameter [{0}]"), + E0309(XLog.STD, "Invalid parameter value, [{0}] = [{1}], {2}"), E0401(XLog.STD, "Missing configuration property [{0}]"), E0402(XLog.STD, "Invalid callback ID [{0}]"), diff --git a/core/src/main/java/org/apache/oozie/coord/CoordUtils.java b/core/src/main/java/org/apache/oozie/coord/CoordUtils.java index 5c08210628..5da08621c1 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordUtils.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordUtils.java @@ -22,15 +22,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.Map; -import java.util.HashMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.commons.lang3.Range; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.CoordinatorActionBean; @@ -63,6 +65,7 @@ public class CoordUtils { + private static final XLog LOG = XLog.getLog(CoordUtils.class); public static final String HADOOP_USER = "user.name"; public static String getDoneFlag(Element doneFlagElement) { @@ -92,7 +95,7 @@ public static Configuration getHadoopConf(Configuration jobConf) { * @throws CommandException thrown if failed to get coordinator actions by given date range */ public static List getCoordActions(String rangeType, String jobId, String scope, - boolean active) throws CommandException { + boolean active) throws CommandException { List coordActions = null; if (rangeType.equals(RestConstants.JOB_COORD_SCOPE_DATE)) { coordActions = CoordUtils.getCoordActionsFromDates(jobId, scope, active); @@ -196,52 +199,116 @@ public static Set getActionsIds(String jobId, String scope) throws Comma ParamChecker.notEmpty(jobId, "jobId"); ParamChecker.notEmpty(scope, "scope"); - Set actions = new LinkedHashSet(); - String[] list = scope.split(","); - for (String s : list) { - s = s.trim(); - // An action range is specified with two actions separated by '-' - if (s.contains("-")) { - String[] range = s.split("-"); - // Check the format for action's range - if (range.length != 2) { - throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s + "', an example of" - + " correct format is 1-5"); - } - int start; - int end; - //Get the starting and ending action numbers - try { - start = Integer.parseInt(range[0].trim()); - } catch (NumberFormatException ne) { - throw new CommandException(ErrorCode.E0302, "could not parse " + range[0].trim() + "into an integer", ne); - } - try { - end = Integer.parseInt(range[1].trim()); - } catch (NumberFormatException ne) { - throw new CommandException(ErrorCode.E0302, "could not parse " + range[1].trim() + "into an integer", ne); - } - if (start > end) { - throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s + "', starting action" - + "number of the range should be less than ending action number, an example will be 1-4"); - } - // Add the actionIds - for (int i = start; i <= end; i++) { - actions.add(jobId + "@" + i); + final Set actions = new LinkedHashSet<>(); + for (final Range range : parseScopeToRanges(scope)) { + // Add the actionIds + for (int i = range.getMinimum(); i <= range.getMaximum(); i++) { + final String jobIdToAdd = jobId + "@" + i; + if (LOG.isTraceEnabled()) { + LOG.trace("Adding {0} to actionSet.", jobIdToAdd); } + actions.add(jobIdToAdd); } - else { + } + return actions; + } + + /** + * Parses a string value into a {@link org.apache.commons.lang3.Range} object while does sanity checking. + * + * @param rangeToParse string representing a minimum and maximum value separated by a dash: '1-5' + * @return the parsed range class + * @throws CommandException when the provided range string is empty, invalid or starting value is greater than ending value + */ + public static Range parseRange(final String rangeToParse) throws CommandException { + final String range = StringUtils.stripToNull(rangeToParse); + if (range == null) { + throw new CommandException(ErrorCode.E0302, String.format("Range cannot be empty: %s", rangeToParse)); + } + final String[] parts = range.split("-"); + if (parts.length != 2) { + throw new CommandException( + ErrorCode.E0302, + String.format("format is wrong for action's range '%s', an example of correct format is 1-5", rangeToParse) + ); + } + try { + final int start = Integer.parseInt(parts[0].trim()); + final int end = Integer.parseInt(parts[1].trim()); + + if (start > end) { + throw new CommandException( + ErrorCode.E0302, + String.format("format is wrong for action's range '%s', starting action number of the range " + + "should be less than ending action number, an example will be 1-4", rangeToParse) + ); + } + + return Range.between(start, end); + } catch (final NumberFormatException ne) { + throw new CommandException( + ErrorCode.E0302, + String.format("could not parse boundaries of %s into an integer", rangeToParse), + ne + ); + } + } + + /** + * Parses a scope definition (comma-separated list of values and ranges) into a list of + * {@link org.apache.commons.lang3.Range}s. If a single value is defined, a [1-1] range will be created. + * + * @param scope comma-separated list of values and ranges + * @return list of ranges based on the provided 'scope' + * @throws CommandException when provided scope string is empty, invalid + * or the ranges' starting value is greater than ending value + */ + public static List> parseScopeToRanges(final String scope) throws CommandException { + if (StringUtils.stripToNull(scope) == null) { + throw new CommandException(ErrorCode.E0302, "scope should not be empty"); + } + final List> result = new ArrayList<>(); + final String[] list = scope.split(","); + for (final String s : list) { + final String range = StringUtils.stripToNull(s); + if (range == null) { + continue; + } + + if (range.contains("-")) { + // assume it is a valid range: 1-5 + result.add(parseRange(range)); + } else { + // assume it is a plain number try { - Integer.parseInt(s); - } - catch (NumberFormatException ne) { - throw new CommandException(ErrorCode.E0302, "format is wrong for action id'" + s - + "'. Integer only."); + final int elem = Integer.parseInt(range); + result.add(Range.between(elem, elem)); + } catch (final NumberFormatException ne) { + throw new CommandException(ErrorCode.E0302, String.format("could not parse %s into an integer", range), ne); } - actions.add(jobId + "@" + s); } } - return actions; + if (LOG.isDebugEnabled()) { + LOG.debug( + "Created the following ranges from \"{0}\": {1}", + scope, + result + .stream() + .map(r -> String.format("[%s-%s]", r.getMinimum(), r.getMaximum())) + .collect(Collectors.joining(", ")) + ); + } + return result; + } + + /** + * Calculates the number of elements described in the list of ranges. + * + * @param ranges list of {@link org.apache.commons.lang3.Range}s + * @return the number of elements the provided ranges contain + */ + public static int getElemCountOfRanges(final List> ranges) { + return ranges.stream().mapToInt(r -> r.getMaximum() - r.getMinimum() + 1).sum(); } /** diff --git a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java index 869568f500..9176fe5827 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java @@ -28,11 +28,28 @@ import javax.servlet.http.HttpServletResponse; import com.google.common.base.Strings; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.oozie.*; +import org.apache.oozie.BaseEngineException; +import org.apache.oozie.BundleEngine; +import org.apache.oozie.BundleEngineException; +import org.apache.oozie.CoordinatorActionBean; +import org.apache.oozie.CoordinatorActionInfo; +import org.apache.oozie.CoordinatorEngine; +import org.apache.oozie.CoordinatorEngineException; +import org.apache.oozie.CoordinatorJobBean; +import org.apache.oozie.DagEngine; +import org.apache.oozie.DagEngineException; +import org.apache.oozie.ErrorCode; +import org.apache.oozie.WorkflowActionBean; +import org.apache.oozie.WorkflowJobBean; +import org.apache.oozie.XException; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; -import org.apache.oozie.client.rest.*; +import org.apache.oozie.client.rest.JsonBean; +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.client.rest.JsonUtils; +import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; import org.apache.oozie.coord.CoordUtils; import org.apache.oozie.service.BundleEngineService; @@ -42,6 +59,7 @@ import org.apache.oozie.service.Services; import org.apache.oozie.service.UUIDService; import org.apache.oozie.util.Instrumentation; +import org.apache.oozie.util.ParameterVerifierException; import org.apache.oozie.util.graph.GraphGenerator; import org.apache.oozie.util.XLog; import org.apache.oozie.util.graph.GraphRenderer; @@ -568,6 +586,8 @@ private JSONObject killCoordinator(HttpServletRequest request, HttpServletRespon String rangeType = request.getParameter(RestConstants.JOB_COORD_RANGE_TYPE_PARAM); String scope = request.getParameter(RestConstants.JOB_COORD_SCOPE_PARAM); + validateScopeSize(scope, rangeType); + try { if (rangeType != null && scope != null) { XLog.getLog(getClass()).info( @@ -730,6 +750,8 @@ private JSONObject reRunCoordinatorActions(HttpServletRequest request, HttpServl "Rerun coordinator for jobId=" + jobId + ", rerunType=" + rerunType + ",scope=" + scope + ",refresh=" + refresh + ", noCleanup=" + noCleanup); + validateScopeSize(scope, rerunType); + try { if (!(rerunType.equals(RestConstants.JOB_COORD_SCOPE_DATE) || rerunType .equals(RestConstants.JOB_COORD_SCOPE_ACTION))) { @@ -753,6 +775,39 @@ private JSONObject reRunCoordinatorActions(HttpServletRequest request, HttpServl return json; } + /** + * Validates if the number of elements defined in 'scope' (comma separated list of values and ranges) + * is less than or equal than the maximum allowed count: oozie.coord.actions.scope.max.size. + * + * @param scope comma separated list of values and ranges + * @throws XServletException if there are too many elements or ranges/values are invalid + */ + static void validateScopeSize(final String scope, final String rangeType) throws XServletException { + if (!"action".equalsIgnoreCase(StringUtils.stripToNull(rangeType))) { + return; + } + + final int maxElemCount = ConfigurationService.getInt("oozie.coord.actions.scope.max.size"); + + try { + final int elemCountOfRanges = CoordUtils.getElemCountOfRanges(CoordUtils.parseScopeToRanges(scope)); + if (elemCountOfRanges > maxElemCount) { + throw new ParameterVerifierException( + ErrorCode.E0309, + "scope", + scope, + String.format( + "too many elements are requested: %s, maximum allowed: %s", + elemCountOfRanges, + maxElemCount + ) + ); + } + } catch (final XException e) { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, e); + } + } + /** * Get workflow job * @@ -1103,6 +1158,9 @@ protected JSONObject getJobsByParentId(HttpServletRequest request, HttpServletRe String coordActionId; String type = request.getParameter(RestConstants.JOB_COORD_RANGE_TYPE_PARAM); String scope = request.getParameter(RestConstants.JOB_COORD_SCOPE_PARAM); + + validateScopeSize(scope, type); + // for getting allruns for coordinator action - 2 alternate endpoints if (type != null && type.equals(RestConstants.JOB_COORD_SCOPE_ACTION) && scope != null) { // endpoint - oozie/v2/coord-job-id?type=action&scope=action-num&show=allruns diff --git a/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java index 1426883794..6f7a75a5dc 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java @@ -30,13 +30,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.oozie.BaseEngine; import org.apache.oozie.BaseEngineException; -import org.apache.oozie.BundleEngine; import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.CoordinatorActionInfo; import org.apache.oozie.CoordinatorEngine; import org.apache.oozie.CoordinatorEngineException; import org.apache.oozie.CoordinatorWfActionBean; -import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.DagEngine; import org.apache.oozie.DagEngineException; import org.apache.oozie.ErrorCode; @@ -46,7 +44,6 @@ import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; import org.apache.oozie.command.coord.CoordCommandUtils; -import org.apache.oozie.command.wf.ActionXCommand; import org.apache.oozie.dependency.ActionDependency; import org.apache.oozie.service.BundleEngineService; import org.apache.oozie.service.CoordinatorEngineService; @@ -222,6 +219,9 @@ private JSONObject ignoreCoordinatorJob(HttpServletRequest request, HttpServletR String type = request.getParameter(RestConstants.JOB_COORD_RANGE_TYPE_PARAM); String scope = request.getParameter(RestConstants.JOB_COORD_SCOPE_PARAM); String changeValue = "status=" + CoordinatorAction.Status.IGNORED; + + validateScopeSize(scope, type); + List coordActions = new ArrayList(); try { if (type != null && !type.equals(RestConstants.JOB_COORD_SCOPE_ACTION)) { diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 01c1095fe0..98d87c5e37 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -2515,6 +2515,17 @@ will be the requeue interval for the actions which are waiting for a long time w + + oozie.coord.actions.scope.max.size + 50 + + Specifies the maximum number of actions that can be queried using the scope parameter in requests. + The 'scope' is a comma-separated list of values and ranges, such as: 1-3,6,7-11,13. + This setting limits the number of actions requested when ignoring, killing, re-running Coordinator actions, + or generating coordinator child job IDs. + + + oozie.validate.ForkJoin diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordUtilsNoServices.java b/core/src/test/java/org/apache/oozie/coord/TestCoordUtilsNoServices.java new file mode 100644 index 0000000000..b08b53c987 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/coord/TestCoordUtilsNoServices.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.coord; + +import org.apache.commons.lang3.Range; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.oozie.command.CommandException; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class TestCoordUtilsNoServices { + + static { + // only to check log messages are printed correctly + Logger logger = Logger.getLogger(CoordUtils.class); + logger.setLevel(Level.TRACE); + } + + @Test + public void testEmptyScopeIsInvalid() { + try { + CoordUtils.parseScopeToRanges(""); + fail("Empty scope should be invalid"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [scope should not be empty]", + e.getMessage() + ); + } + + try { + CoordUtils.parseScopeToRanges(" "); + fail("Empty scope should be invalid"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [scope should not be empty]", + e.getMessage() + ); + } + + try { + CoordUtils.parseScopeToRanges("1-3,-,5-9"); + fail("Range without boundaries should be invalid"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [format is wrong for action's range '-', " + + "an example of correct format is 1-5]", + e.getMessage() + ); + } + } + + @Test + public void testScopeToRanges() throws CommandException { + final List> result = CoordUtils.parseScopeToRanges(" 1-3, 5, 7-900,1100,2000 -2000"); + assertEquals(Arrays.asList( + Range.between(1, 3), // 3 numbers: 1, 2, 3 + Range.between(5, 5), // 1 number: 5 + Range.between(7, 900), // 894 numbers: 7, 8, ...899, 900 + Range.between(1100, 1100), // 1 number: 1100 + Range.between(2000, 2000) // 1 number: 2000 + ), result); + + assertEquals("Unexpected number of elements in the ranges", + 900, CoordUtils.getElemCountOfRanges(result)); + } + + @Test + public void testScopeRangesInvalidRange() { + try { + CoordUtils.parseScopeToRanges("3-1,5,7-900,11"); + fail("'3-1,5,7-900,11' is should be marked as invalid scope"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [format is wrong for action's range '3-1', " + + "starting action number of the range should be less than ending action number, " + + "an example will be 1-4]", + e.getMessage() + ); + } + } + + @Test + public void testScopeInvalidRangeMin() { + try { + CoordUtils.parseScopeToRanges("-4-0"); + fail("'-4-0' is should be marked as invalid scope"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [format is wrong for action's range '-4-0', " + + "an example of correct format is 1-5]", + e.getMessage() + ); + } + } + + @Test + public void testScopeInvalidRangeMax() { + try { + CoordUtils.parseScopeToRanges("30-A"); + fail("'30-A' is should be marked as invalid scope"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [could not parse boundaries of 30-A into an integer]", + e.getMessage() + ); + } + } + + @Test + public void testScopeInvalidNumberAmongRanges() { + try { + CoordUtils.parseScopeToRanges("2-7,A,9-11"); + fail("'2-7,A,9-11' is should be marked as invalid scope"); + } catch (CommandException e) { + assertEquals( + "Unexpected error message", + "E0302: Invalid parameter [could not parse A into an integer]", + e.getMessage() + ); + } + } + +} diff --git a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java b/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java index 2d1bc1bc9d..a662c0358a 100644 --- a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java +++ b/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java @@ -64,11 +64,21 @@ protected URL createURL(String resource, Map parameters) throws @SuppressWarnings("unchecked") protected void runTest(String servletPath, Class servletClass, boolean securityEnabled, Callable assertions) throws Exception { - runTest(new String[]{servletPath}, new Class[]{servletClass}, securityEnabled, assertions); + runTest(servletPath, servletClass, securityEnabled, assertions, null); + } + + protected void runTest(String servletPath, Class servletClass, boolean securityEnabled, Callable assertions, + Map extraServicesConf) throws Exception { + runTest(new String[]{servletPath}, new Class[]{servletClass}, securityEnabled, assertions, extraServicesConf); } protected void runTest(String[] servletPath, Class[] servletClass, boolean securityEnabled, Callable assertions) throws Exception { + runTest(servletPath,servletClass, securityEnabled, assertions, null); + } + + protected void runTest(String[] servletPath, Class[] servletClass, boolean securityEnabled, + Callable assertions, Map extraServicesConf) throws Exception { Services services = new Services(); this.servletPath = servletPath[0]; try { @@ -79,6 +89,11 @@ protected void runTest(String[] servletPath, Class[] servletClass, boolean secur ProxyUserService.GROUPS, "*"); services.init(); services.getConf().setBoolean(AuthorizationService.CONF_SECURITY_ENABLED, securityEnabled); + + if (extraServicesConf != null && !extraServicesConf.isEmpty()) { + extraServicesConf.forEach(Services.get().getConf()::set); + } + Services.get().setService(ForTestAuthorizationService.class); Services.get().setService(ForTestWorkflowStoreService.class); Services.get().setService(MockDagEngineService.class); diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV1JobServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV1JobServlet.java index 27e88ef286..edd3f95a58 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV1JobServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV1JobServlet.java @@ -430,4 +430,133 @@ public Void call() throws Exception { } }); } + + public void testCoordActionKillWithScopeValidation() throws Exception { + runTest("/v1/job/*", V1JobServlet.class, IS_SECURITY_ENABLED, () -> { + MockCoordinatorEngineService.reset(); + final Map params = new HashMap<>(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_KILL); + params.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, "action"); + params.put(RestConstants.JOB_COORD_SCOPE_PARAM, "1-300"); + + final URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + + // conn.getResponseCode() is also needed to send the request + final int responseCode = conn.getResponseCode(); + final String error = conn.getHeaderField(RestConstants.OOZIE_ERROR_CODE); + final String message = conn.getHeaderField(RestConstants.OOZIE_ERROR_MESSAGE); + + assertEquals("Unexpected error code: " + conn.getResponseMessage(), + HttpServletResponse.SC_BAD_REQUEST, responseCode); + assertEquals("Unexpected Oozie error code", "E0309", error); + assertEquals( + "Unexpected error message", + "E0309: Invalid parameter value, [scope] = [1-300], " + + "too many elements are requested: 300, maximum allowed: 50", + message + ); + + + return null; + }); + } + + public void testCoordActionRerunWithScopeValidation() throws Exception { + runTest("/v1/job/*", V1JobServlet.class, IS_SECURITY_ENABLED, () -> { + MockCoordinatorEngineService.reset(); + final Map params = new HashMap<>(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_RERUN); + params.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, "action"); + params.put(RestConstants.JOB_COORD_SCOPE_PARAM, "1-300"); + + final URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + new Configuration().writeXml(conn.getOutputStream()); + + // conn.getResponseCode() is also needed to send the request + final int responseCode = conn.getResponseCode(); + final String error = conn.getHeaderField(RestConstants.OOZIE_ERROR_CODE); + final String message = conn.getHeaderField(RestConstants.OOZIE_ERROR_MESSAGE); + + assertEquals("Unexpected error code: " + conn.getResponseMessage(), + HttpServletResponse.SC_BAD_REQUEST, responseCode); + assertEquals("Unexpected Oozie error code", "E0309", error); + assertEquals( + "Unexpected error message", + "E0309: Invalid parameter value, [scope] = [1-300], " + + "too many elements are requested: 300, maximum allowed: 50", + message + ); + + return null; + }); + } + + public void testCoordActionShowWithScopeValidation() throws Exception { + runTest("/v1/job/*", V1JobServlet.class, IS_SECURITY_ENABLED, () -> { + MockCoordinatorEngineService.reset(); + final Map params = new HashMap<>(); + params.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, "action"); + params.put(RestConstants.JOB_COORD_SCOPE_PARAM, "1-300"); + params.put(RestConstants.JOB_SHOW_PARAM, RestConstants.ALL_WORKFLOWS_FOR_COORD_ACTION); + + final URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + + // conn.getResponseCode() is also needed to send the request + final int responseCode = conn.getResponseCode(); + final String error = conn.getHeaderField(RestConstants.OOZIE_ERROR_CODE); + final String message = conn.getHeaderField(RestConstants.OOZIE_ERROR_MESSAGE); + + assertEquals("Unexpected error code: " + conn.getResponseMessage(), + HttpServletResponse.SC_BAD_REQUEST, responseCode); + assertEquals("Unexpected Oozie error code", "E0309", error); + assertEquals( + "Unexpected error message", + "E0309: Invalid parameter value, [scope] = [1-300], " + + "too many elements are requested: 300, maximum allowed: 50", + message + ); + + return null; + }); + } + + public void testCoordActionKillWithScopeValidationIncreasedScope() throws Exception { + final Map extraServicesConf = new HashMap<>(); + extraServicesConf.put("oozie.coord.actions.scope.max.size", "300"); + + runTest("/v1/job/*", V1JobServlet.class, IS_SECURITY_ENABLED, () -> { + MockCoordinatorEngineService.reset(); + final Map params = new HashMap<>(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_KILL); + params.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, "action"); + params.put(RestConstants.JOB_COORD_SCOPE_PARAM, "1-300"); + + final URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + + // conn.getResponseCode() is also needed to send the request + final int responseCode = conn.getResponseCode(); + final String message = conn.getResponseMessage(); + + assertEquals("Unexpected error code: " + message, HttpServletResponse.SC_OK, responseCode); + assertEquals("Unexpected error message", "OK", message); + + return null; + }, extraServicesConf); + } + } diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java index 055dfb15a0..7b242577f3 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java @@ -460,4 +460,41 @@ public Void call() throws Exception { } }); } + + public void testCoordJobIgnoreWithScopeValidation() throws Exception { + runTest("/v2/job/*", V2JobServlet.class, IS_SECURITY_ENABLED, () -> { + + MockDagEngineService.reset(); + final Map params = new HashMap<>(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_IGNORE); + params.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, "action"); + params.put(RestConstants.JOB_COORD_SCOPE_PARAM, "1-300"); + + // url - oozie/v2/coord_job_id?action=ignore&scope=1-300&type=action + final URL url = createURL("0000001-1234567890-C", params); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + + // conn.getResponseCode() is also needed to send the request + final int responseCode = conn.getResponseCode(); + final String error = conn.getHeaderField(RestConstants.OOZIE_ERROR_CODE); + final String message = conn.getHeaderField(RestConstants.OOZIE_ERROR_MESSAGE); + + assertEquals("Unexpected error code: " + conn.getResponseMessage(), + HttpServletResponse.SC_BAD_REQUEST, responseCode); + assertEquals("Unexpected Oozie error code", "E0309", error); + assertEquals( + "Unexpected error message", + "E0309: Invalid parameter value, [scope] = [1-300], " + + "too many elements are requested: 300, maximum allowed: 50", + message + ); + + + return null; + }); + } + }