diff --git a/docs/classes.puml b/docs/classes.puml index 36acf003..586b0deb 100644 --- a/docs/classes.puml +++ b/docs/classes.puml @@ -5,28 +5,7 @@ hide empty members hide circle class JmeterDsl { - DslTestPlan testPlan(TestPlanChild[] children) - DslThreadGroup threadGroup(int threads, int iterations, ThreadGroupChild[] children) - DslThreadGroup threadGroup(String name, int threads, int iterations, ThreadGroupChild[] children) - DslThreadGroup threadGroup(int threads, Duration duration, ThreadGroupChild[] children) - DslThreadGroup threadGroup(String name, int threads, Duration duration, ThreadGroupChild[] children) - DslHttpSampler httpSampler(String url) - DslHttpSampler httpSampler(Function urlSupplier) - DslHttpSampler httpSampler(String name, String url) - DslHttpSampler httpSampler(String name, Function urlSupplier) - HttpHeaders httpHeaders() - DslJsr223PreProcessor jsr223PreProcessor(String script) - DslJsr223PreProcessor jsr223PreProcessor(String name, String script) - DslJsr223PreProcessor jsr223PreProcessor(PreProcessorScript script) - DslJsr223PreProcessor jsr223PreProcessor(String name, PreProcessorScript script) - DslRegexExtractor regexExtractor(String variableName, String regex) - DslJsr223PostProcessor jsr223PostProcessor(String script) - DslJsr223PostProcessor jsr223PostProcessor(String name, String script) - DslJsr223PostProcessor jsr223PostProcessor(PostProcessorScript script) - DslJsr223PostProcessor jsr223PostProcessor(String name, PostProcessorScript script) - JtlWriter jtlWriter(String jtlFile) - InfluxDbBackendListener influxDbListener(String url) - HtmlReporter htmlReporter(String reportDirectory) + .. } package core { @@ -63,6 +42,7 @@ package core { class DslThreadGroup extends TestElementContainer implements TestPlanChild { int threads int iterations + Duration duration } interface ThreadGroupChild extends DslTestElement @@ -71,35 +51,12 @@ package core { interface SamplerChild extends DslTestElement - interface MultiScopedTestElement extends TestPlanChild, ThreadGroupChild, SamplerChild - - package listeners { - - class JtlWriter extends BaseTestElement implements MultiScopedTestElement { - String jtlFilePath - } - - class HtmlReporter extends BaseTestElement implements MultiScopedTestElement { - File reportDirectory - } - - class InfluxDbBackendListener extends BaseTestElement implements MultiScopedTestElement { - String url - String token - String title - int queueSize - InfluxDbBackendListener token(String token) - InfluxDbBackendListener title(String token) - InfluxDbBackendListener queueSize(int queueSize) - } - - } + interface MultiLevelTestElement extends TestPlanChild, ThreadGroupChild, SamplerChild abstract class DslJsr223TestElement extends BaseTestElement { - String script - String language - DslJsr223TestElement language(String language) - abstract DslJsr223TestElement buildJsr223TestElement() + String script + String language + abstract DslJsr223TestElement buildJsr223TestElement() } interface Jsr223Script { @@ -115,9 +72,19 @@ package core { String Label } + package preprocessors { + + class DslJsr223PreProcessor extends DslJsr223TestElement implements MultiLevelTestElement + + interface PreProcessorScript extends Jsr223Script + + class PreProcessorVars extends Jsr223ScriptVars + + } + package postprocessors { - class DslJsr223PostProcessor extends DslJsr223TestElement implements MultiScopedTestElement + class DslJsr223PostProcessor extends DslJsr223TestElement implements MultiLevelTestElement interface PostProcessorScript extends Jsr223Script @@ -125,7 +92,7 @@ package core { SampleResult prev } - class DslRegexExtractor extends BaseTestElement implements MultiScopedTestElement { + class DslRegexExtractor extends BaseTestElement implements MultiLevelTestElement { String variableName String regex int matchNumber @@ -134,12 +101,6 @@ package core { TargetField fieldToCheck Scope scope String scopeVariable - DslRegexExtractor matchNumber(int matchNumber) - DslRegexExtractor template(String template) - DslRegexExtractor defaultValue(String defaultValue) - DslRegexExtractor fieldToCheck(TargetField fieldToCheck) - DslRegexExtractor scope(Scope scope) - DslRegexExtractor scopeVariable(String scopeVariable) } enum TargetField { @@ -164,13 +125,52 @@ package core { } - package preprocessors { + package assertions { + class DslResponseAssertion extends BaseTestElement implements MultiLevelTestElement { + TargetField fieldToTest + boolean ignoreStatus + String[] testStrings + TestStringStrategy testStrategy + boolean invertCheck + boolean anyMatch + } - class DslJsr223PreProcessor extends DslJsr223TestElement implements MultiScopedTestElement + enum TargetField { + RESPONSE_BODY + RESPONSE_BODY_AS_DOCUMENT + RESPONSE_CODE + RESPONSE_MESSAGE + RESPONSE_HEADERS + REQUEST_HEADERS + REQUEST_URL + REQUEST_BODY + } - interface PreProcessorScript extends Jsr223Script + enum TestStringStrategy { + SUBSTRING + EQUALS + CONTAINS_REGEX + MATCHES_REGEX + } - class PreProcessorVars extends Jsr223ScriptVars + } + + package listeners { + + class JtlWriter extends BaseTestElement implements MultiLevelTestElement { + String jtlFilePath + } + + class HtmlReporter extends BaseTestElement implements MultiLevelTestElement { + File reportDirectory + } + + class InfluxDbBackendListener extends BaseTestElement implements MultiLevelTestElement { + String url + String token + String title + int queueSize + } } @@ -182,18 +182,10 @@ package http { String url HttpMethod method String body - DslHttpSampler post(String body, Type contentType) - DslHttpSampler post(Function bodySupplier, Type contentType) - DslHttpSampler method(HttpMethod method) - DslHttpSampler body(String body) - DslHttpSampler body(Function bodySupplier) - DslHttpSampler header(String name, String value) - DslHttpSampler header(String name, Function valueSupplier) - DslHttpSampler children(SamplerChild[] children) } - class HttpHeaders extends BaseTestElement implements MultiScopedTestElement { - HttpHeaders header(String name, String value) + class HttpHeaders extends BaseTestElement implements MultiLevelTestElement { + Map headers } } diff --git a/docs/use-cases.md b/docs/use-cases.md index 771c2278..c8bfa9af 100644 --- a/docs/use-cases.md +++ b/docs/use-cases.md @@ -63,6 +63,82 @@ Check [BlazeMeterEngine](../jmeter-java-dsl-blazemeter/src/main/java/us/abstract > **Warning:** If you use JSR223 Pre or Post processors with Java code (lambdas) instead of strings, or use one of the HTTP Sampler methods which receive a function as parameter, then BlazeMeter execution won't work. You can migrate them to use jsrPreProcessor with string scripts instead. Check for these methods documentation for more details. +## Check for expected response + +By default, JMeter marks any HTTP request with a fail response code (4xx or 5xx) as failed, which allows you to easily identify when some request unexpectedly fails. But in many cases this is not enough or desirable, and you need to check for response body (or some other field) to contain (or not contain) certain string. + +This is usually accomplished with the usage of Response Assertions, which provides an easy and fast way to verify that you get the proper response for each step of the test plan, marking the request as failure when such condition is not met. + +Here is an example on how to specify a response assertion in jmeter-java-dsl: + +```java +import static org.assertj.core.api.Assertions.assertThat; +import static us.abstracta.jmeter.javadsl.JmeterDsl.*; + +import java.io.IOException; +import java.time.Duration; +import org.eclipse.jetty.http.MimeTypes.Type; +import org.junit.jupiter.api.Test; + +public class PerformanceTest { + + @Test + public void testPerformance() throws IOException { + TestPlanStats stats = testPlan( + threadGroup(2, 10, + httpSampler("http://my.service") + .children( + responseAssertion().containsSubstrings("OK") + ) + ) + ).run(); + assertThat(stats.overall().elapsedTimePercentile99()).isLessThan(Duration.ofSeconds(5)); + } + +} +``` + +Check [Response Assertion](../jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertion.java) for more details and additional options + +For more complex scenarios you can use (for the time being) [later mentioned JSR223 Post processor](#change-sample-result-statuses-with-custom-logic). + +## Use part of a response in a following request + +It is a usual requirement while creating a test plan for an application, to be able to use part of a response (e.g.: a generated ID, token, etc) in a subsequent request. This can be easily achieved using JMeter extractors and variables. Here is an example with jmeter-java-dsl: + +```java +import static org.assertj.core.api.Assertions.assertThat; +import static us.abstracta.jmeter.javadsl.JmeterDsl.*; + +import java.io.IOException; +import java.time.Duration; +import org.eclipse.jetty.http.MimeTypes.Type; +import org.junit.jupiter.api.Test; + +public class PerformanceTest { + + @Test + public void testPerformance() throws IOException { + TestPlanStats stats = testPlan( + threadGroup(2, 10, + httpSampler("http://my.service/accounts") + .post("{\"name\": \"John Doe\"}", Type.APPLICATION_JSON) + .children( + regexExtractor("ACCOUNT_ID", "\"id\":\"([^\"]+)\"") + ), + httpSampler("http://my.service/accounts/${ACCOUNT_ID}") + ) + ).run(); + assertThat(stats.overall().elapsedTimePercentile99()).isLessThan(Duration.ofSeconds(5)); + } + +} +``` + +Check [DslRegexExtractor](../jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslRegexExtractor.java) for more details and additional options. + +For more complex scenarios you can use [later mentioned JSR223 Post processor](#change-sample-result-statuses-with-custom-logic). + ## Save as JMX In case you want to load a test plan in JMeter GUI, you can save it just invoking `saveAsJMX` method in the test plan as in following example: @@ -142,9 +218,9 @@ This can be used to just run existing JMX files, or when DSL has no support for > ``` ## Publish test metrics to [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) and visualizing them in [Grafana](https://grafana.com/) - + When running tests with JMeter (and in particular with jmeter-java-dsl) a usual requirement is to be able to store such test runs in a persistent database to later on review such metrics, and compare different test runs. Additionally, jmeter-java-dsl only provides some summary data of test run in the console while it is running, but, since it doesn't provide any sort of UI, doesn't allow to easily analyze such information as it can be done in JMeter GUI. - + To overcome these limitations you can use provided support for publishing JMeter test run metrics to InfluxDB, which allows keeping record of all run statistics and, through Grafana, get some nice dashboards like the following one: ![grafana](influxdb/grafana.png) @@ -269,8 +345,8 @@ jsr223PostProcessor(s -> { Check [DslJsr223PostProcessor](../jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslJsr223PostProcessor.java) for more details and additional options. -JSR223PostProcessor is a very powerful tool, but is not the only, nor the best, alternative for many cases where JMeter already provides a better and simpler alternative (eg: asserting response bodies contain some string). Currently, jmeter-java-dsl does not support all the features JMeter provides. So, if you need something already provided by JMeter, please create an issue in GitHub requesting such a feature or submit a pull request with the required support. - +**Note:** JSR223PostProcessor is a very powerful tool, but is not the only, nor the best, alternative for many cases where JMeter already provides a better and simpler alternative. For instance, previously mentioned might be implemented with previously presented [Response Assertion](#check-for-expected-response). + ## Provide Request Parameters Programmatically per Request With the standard DSL you can provide static values to request parameters, such as a body. However, you may also want to be able to modify your requests for each call. This is common in cases where your request creates something that must have unique values. @@ -329,40 +405,3 @@ post(s -> buildRequestBody(s.vars), Type.TEXT_PLAIN) > **WARNING:** using java code (lambdas) will only work with embedded JMeter engine (no support for saving to JMX and running it in JMeter GUI, or running it with BlazeMeter). Use the first option to avoid such limitations. Check [DslJsr223PreProcessor](../jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/preprocessors/DslJsr223PreProcessor.java) & [DslHttpSampler](../jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/http/DslHttpSampler.java) for more details and additional options. - -## Use part of a response in a following request - -It is a usual requirement while creating a test plan for an application, to be able to use part of a response (e.g.: a generated ID, token, etc) in a subsequent request. This can be easily achieved using JMeter extractors and variables. Here is an example with jmeter-java-dsl: - -```java -import static org.assertj.core.api.Assertions.assertThat; -import static us.abstracta.jmeter.javadsl.JmeterDsl.*; - -import java.io.IOException; -import java.time.Duration; -import org.eclipse.jetty.http.MimeTypes.Type; -import org.junit.jupiter.api.Test; - -public class PerformanceTest { - - @Test - public void testPerformance() throws IOException { - TestPlanStats stats = testPlan( - threadGroup(2, 10, - httpSampler("http://my.service/accounts") - .post("{\"name\": \"John Doe\"}", Type.APPLICATION_JSON) - .children( - regexExtractor("ACCOUNT_ID", "\"id\":\"([^\"]+)\"") - ), - httpSampler("http://my.service/accounts/${ACCOUNT_ID}") - ) - ).run(); - assertThat(stats.overall().elapsedTimePercentile99()).isLessThan(Duration.ofSeconds(5)); - } - -} -``` - -Check [DslRegexExtractor](../jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslRegexExtractor.java) for more details and additional options. - -For more complex scenarios you can use [previously mentioned JSR223 Post processor](#change-sample-result-statuses-with-custom-logic). diff --git a/jmeter-java-dsl-blazemeter/src/main/java/us/abstracta/jmeter/javadsl/blazemeter/BlazeMeterEngine.java b/jmeter-java-dsl-blazemeter/src/main/java/us/abstracta/jmeter/javadsl/blazemeter/BlazeMeterEngine.java index 73b13537..f0a8b703 100644 --- a/jmeter-java-dsl-blazemeter/src/main/java/us/abstracta/jmeter/javadsl/blazemeter/BlazeMeterEngine.java +++ b/jmeter-java-dsl-blazemeter/src/main/java/us/abstracta/jmeter/javadsl/blazemeter/BlazeMeterEngine.java @@ -44,9 +44,11 @@ public class BlazeMeterEngine implements DslJmeterEngine { private boolean useDebugRun; /** - * @param authToken is the authentication token to be used to access BlazeMeter API.

It follows - * the following format: <Key ID>:<Key Secret>.

Check BlazeMeter + * @param authToken is the authentication token to be used to access BlazeMeter API. + *

+ * It follows the following format: <Key ID>:<Key Secret>. + *

+ * Check BlazeMeter * API keys for instructions on how to generate them. */ public BlazeMeterEngine(String authToken) { diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/JmeterDsl.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/JmeterDsl.java index 9a6bab4c..fcf38e49 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/JmeterDsl.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/JmeterDsl.java @@ -10,6 +10,7 @@ import us.abstracta.jmeter.javadsl.core.DslTestPlan.TestPlanChild; import us.abstracta.jmeter.javadsl.core.DslThreadGroup; import us.abstracta.jmeter.javadsl.core.DslThreadGroup.ThreadGroupChild; +import us.abstracta.jmeter.javadsl.core.assertions.DslResponseAssertion; import us.abstracta.jmeter.javadsl.core.listeners.HtmlReporter; import us.abstracta.jmeter.javadsl.core.listeners.InfluxDbBackendListener; import us.abstracta.jmeter.javadsl.core.listeners.JtlWriter; @@ -259,6 +260,10 @@ public static DslJsr223PreProcessor jsr223PreProcessor(String name, PreProcessor * expression between parenthesis) matched by the regular expression. * * @param variableName is the name of the variable to be used to store the extracted value to. + * Additional variables {@code _g} will be created for each regular + * expression capturing group (segment of regex between parenthesis), being group 0 the entire + * match of the regex. {@code _g} variable contains the number of matched capturing + * groups (not counting the group 0). * @param regex regular expression used to extract part of request or response. * @return the Regex Extractor which can be used to define additional settings to use when * extracting (like defining match number, template, etc). @@ -333,6 +338,43 @@ public static DslJsr223PostProcessor jsr223PostProcessor(String name, return new DslJsr223PostProcessor(name, script); } + /** + * Builds a Response Assertion to be able to check that obtained sample result is the expected + * one. + * + * JMeter by default uses repose codes (eg: 4xx and 5xx HTTP response codes are error codes) to + * determine if a request was success or not, but in some cases this might not be enough or + * correct. In some cases applications might not behave in this way, for example, they might + * return a 200 HTTP status code but with an error message in the body, or the response might be a + * success one, but the information contained within the response is not the expected one to + * continue executing the test. In such scenarios you can use response assertions to properly + * verify your assumptions before continuing with next request in the test plan. + * + * By default response assertion will use the response body of the main sample result (not sub + * samples as redirects, or embedded resources) to check the specified criteria (substring match, + * entire string equality, contained regex or entire regex match) against. + * + * @return the create Response Assertion which should be modified to apply the proper criteria. + * Check {@link DslResponseAssertion} for all available options. + * @see DslResponseAssertion + */ + public static DslResponseAssertion responseAssertion() { + return new DslResponseAssertion(null); + } + + /** + * Same as {@link #responseAssertion()} but allowing to set a name on the assertion, which can be + * later used to identify assertion results and differentiate it from other assertions. + * + * @param name is the name to be assigned to the assertion + * @return the create Response Assertion which should be modified to apply the proper criteria. + * Check {@link DslResponseAssertion} for all available options. + * @see #responseAssertion(String) + */ + public static DslResponseAssertion responseAssertion(String name) { + return new DslResponseAssertion(name); + } + /** * Builds a Simple Data Writer to write all collected results to a JTL file. * diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/DslScopedTestElement.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/DslScopedTestElement.java new file mode 100644 index 00000000..cd7d0ae9 --- /dev/null +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/DslScopedTestElement.java @@ -0,0 +1,87 @@ +package us.abstracta.jmeter.javadsl.core; + +import java.util.function.Consumer; +import org.apache.jmeter.gui.JMeterGUIComponent; +import org.apache.jmeter.testelement.AbstractScopedTestElement; + +/** + * Contains common logic for test elements that only process certain samples. + * + * @param is the type of the test element that extends this class (to properly inherit fluent + * API methods). + */ +public abstract class DslScopedTestElement extends BaseTestElement { + + private Scope scope = Scope.MAIN_SAMPLE; + private String scopeVariable; + + protected DslScopedTestElement(String name, Class guiClass) { + super(name, guiClass); + } + + /** + * Allows specifying if the extractor should be specified to main sample and/or sub samples. + *

+ * When not specified the regular extractor will only apply to main sample. + * + * @param scope specifying to what sample result apply the regular extractor to. + * @return the DslRegexExtractor to allow fluent usage and setting other properties. + * @see Scope + */ + public T scope(Scope scope) { + this.scope = scope; + return (T) this; + } + + /** + * Allows specifying that the regular extractor should be applied to the contents of a given + * JMeter variable. + *

+ * This setting overrides any setting on scope and fieldToCheck. + * + * @param scopeVariable specifies the name of the variable to apply the regular extractor to. + * @return the DslRegexExtractor to allow fluent usage and setting other properties. + */ + public T scopeVariable(String scopeVariable) { + this.scopeVariable = scopeVariable; + return (T) this; + } + + protected void setScopeTo(AbstractScopedTestElement testElement) { + scope.applyTo(testElement); + if (scopeVariable != null) { + testElement.setScopeVariable(scopeVariable); + } + } + + /** + * Specifies to which samples apply the regular extractor to. + */ + public enum Scope { + /** + * Applies the regular extractor to all samples (main and sub samples). + */ + ALL_SAMPLES(AbstractScopedTestElement::setScopeAll), + /** + * Applies the regular extractor only to main sample (sub samples, like redirects, are not + * included). + */ + MAIN_SAMPLE(AbstractScopedTestElement::setScopeParent), + /** + * Applies the regular extractor only to sub samples (redirects, embedded resources, etc). + */ + SUB_SAMPLES(AbstractScopedTestElement::setScopeChildren); + + private final Consumer applier; + + Scope(Consumer applier) { + this.applier = applier; + } + + private void applyTo(AbstractScopedTestElement re) { + applier.accept(re); + } + + } + +} diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/MultiScopedTestElement.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/MultiLevelTestElement.java similarity index 82% rename from jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/MultiScopedTestElement.java rename to jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/MultiLevelTestElement.java index 91060c12..2a2e24ef 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/MultiScopedTestElement.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/MultiLevelTestElement.java @@ -8,6 +8,6 @@ * This is just a simple interface to avoid code duplication for test elements that apply at * different levels of a test plan (at test plan, thread group or as sampler child). */ -public interface MultiScopedTestElement extends TestPlanChild, ThreadGroupChild, SamplerChild { +public interface MultiLevelTestElement extends TestPlanChild, ThreadGroupChild, SamplerChild { } diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertion.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertion.java new file mode 100644 index 00000000..df92c5b7 --- /dev/null +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertion.java @@ -0,0 +1,301 @@ +package us.abstracta.jmeter.javadsl.core.assertions; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import org.apache.jmeter.assertions.ResponseAssertion; +import org.apache.jmeter.assertions.gui.AssertionGui; +import org.apache.jmeter.testelement.TestElement; +import us.abstracta.jmeter.javadsl.core.DslScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; + +public class DslResponseAssertion extends DslScopedTestElement implements + MultiLevelTestElement { + + private TargetField fieldToTest = TargetField.RESPONSE_BODY; + private boolean ignoreStatus; + private List testStrings = new ArrayList<>(); + private TestStringStrategy testStrategy = TestStringStrategy.SUBSTRING; + private boolean invertCheck; + private boolean anyMatch; + + public DslResponseAssertion(String name) { + super(name != null ? name : "Response Assertion", AssertionGui.class); + } + + /** + * Specifies what field to apply the assertion to. + *

+ * When not specified it will apply the given assertion to the response body. + * + * @param fieldToTest specifies the field to apply the assertion to. + * @return the Response Assertion to allow configuring other potential settings in a fluent API + * way. + * @see TargetField + */ + public DslResponseAssertion fieldToTest(TargetField fieldToTest) { + this.fieldToTest = fieldToTest; + return this; + } + + /** + * Specifies that any previously status set to the request should be ignored, and request should + * be marked as success by default. + *

+ * This allows overriding the default behavior provided by JMeter when marking requests as failed + * (eg: HTTP status codes like 4xx or 5xx). This is particularly useful when tested application + * returns an unsuccessful response (eg: 400) but you want to consider some of those cases still + * as successful using a different criteria to determine when they are actually a failure (an + * unexpected response). + *

+ * Take into consideration that if you specify multiple response assertions to the same sampler, + * then if this flag is enabled, any previous assertion result in same sampler will be ignored + * (marked as success). So, consider setting this flag in first response assertion only. + * + * @return the Response Assertion to allow configuring other potential settings in a fluent API + * way. + */ + public DslResponseAssertion ignoreStatus() { + this.ignoreStatus = true; + return this; + } + + /** + * Checks if the specified {@link #fieldToTest(TargetField)} contains the given substrings. + *

+ * By default the main sample (not sub samples) response body will be checked, and all supplied + * substrings must be contained. Review other methods in this class if you need to check + * substrings but in some other ways (eg: in response headers, any match is enough, or none of + * specified substrings should be contained). + * + * @param substrings list of strings to be searched in the given field to test (by default + * response body). + * @return the Response Assertion to allow configuring other potential settings in a fluent API + * way. + */ + public DslResponseAssertion containsSubstrings(String... substrings) { + return testStrings(substrings, TestStringStrategy.SUBSTRING); + } + + private DslResponseAssertion testStrings(String[] testStrings, TestStringStrategy strategy) { + this.testStrings = Arrays.asList(testStrings); + this.testStrategy = strategy; + return this; + } + + /** + * Compares the configured {@link #fieldToTest(TargetField)} to the given strings for equality. + *

+ * By default the main sample (not sub samples) response body will be checked, and all supplied + * strings must be equal to the body (in default setting only makes sense to specify one string). + * Review other methods in this class if you need to check equality to entire strings but in some + * other ways (eg: in response headers, any match is enough, or none of specified strings should + * be equal to the field value). + * + * @param strings list of strings to be compared against the given field to test (by default + * response body). + * @return the Response Assertion to allow configuring other potential settings in a fluent API + */ + public DslResponseAssertion equalsToStrings(String... strings) { + return testStrings(strings, TestStringStrategy.EQUALS); + } + + /** + * Checks if the configured {@link #fieldToTest(TargetField)} contains matches for given regular + * expressions. + *

+ * By default the main sample (not sub samples) response body will be checked, and all supplied + * regular expressions must contain a match in the body. Review other methods in this class if you + * need to check regular expressions matches are contained but in some other ways (eg: in response + * headers, any regex match is enough, or none of specified regex should be contained in the field + * value). + *

+ * By default regular expressions evaluate in multi-line mode, which means that '.' does not match + * new lines, '^' matches start of lines and '$' matches end of lines. To use single-line mode + * prefix '(?s)' to the regular expressions. Regular expressions are also by default case + * sensitive, which can be changed to insensitive by adding '(?i)' to the regex. + * + * @param regexes list of regular expressions to search for matches in the field to test (by + * default response body). + * @return the Response Assertion to allow configuring other potential settings in a fluent API + */ + public DslResponseAssertion containsRegexes(String... regexes) { + return testStrings(regexes, TestStringStrategy.CONTAINS_REGEX); + } + + /** + * Checks if the configured {@link #fieldToTest(TargetField)} matches (completely, and not just + * part of it) given regular expressions. + *

+ * By default the main sample (not sub samples) response body will be checked, and all supplied + * regular expressions must match the entire body. Review other methods in this class if you need + * to check regular expressions matches but in some other ways (eg: in response headers, any regex + * match is enough, or none of specified regex should be matched with the field value). + *

+ * By default regular expressions evaluate in multi-line mode, which means that '.' does not match + * new lines, '^' matches start of lines and '$' matches end of lines. To use single-line mode + * prefix '(?s)' to the regular expressions. Regular expressions are also by default case + * sensitive, which can be changed to insensitive by adding '(?i)' to the regex. + * + * @param regexes list of regular expressions the field to test (by default response body) must + * match. + * @return the Response Assertion to allow configuring other potential settings in a fluent API + */ + public DslResponseAssertion matchesRegexes(String... regexes) { + return testStrings(regexes, TestStringStrategy.MATCHES_REGEX); + } + + /** + * Allows to invert/negate each of the checks applied by the assertion. + *

+ * This is the same as the "Not" option in Response Assertion in JMeter GUI. + *

+ * It is important to note that the inversion of the check happens at each check and not to the + * final result. Eg: + * + * 

{@code
+   *   responseAssertion().containsSubstrings("error", "failure").invertCheck()
+   * }
+ * + * Will check that the response does not contain "error" and does not contain "failure". You can + * think it as {@code !(containsSubstring("error")) && !(containsSubstring("failure"))}. + *

+ * Similar logic applies when using in combination with anyMatch method. Eg: + * + * 

{@code
+   *    responseAssertion().containsSubstrings("error", "failure").invertCheck().matchAny()
+   * }
+ * + * Will check that response does not contain both "error" and "failure" at the same time. This is + * analogous to {@code !(containsSubstring("error")) || !(containsSubstring("failure)}, which is + * equivalent to {@code !(containsSubstring("error") && containsSubstring("failure))}. + *

+ * Keep in mind that order of invocations of methods in response assertion is irrelevant (so + * {@code invertCheck().matchAny()} gets the same result as {@code matchAny().invertCheck()}). + * + * @return the Response Assertion to allow configuring other potential settings in a fluent API + */ + public DslResponseAssertion invertCheck() { + this.invertCheck = !this.invertCheck; + return this; + } + + /** + * Specifies that if any check matches then the response assertion is satisfied. + *

+ * This is the same as the "Or" option in Response Assertion in JMeter GUI. + *

+ * By default when you use something like this: + * + * 

{@code
+   *    responseAssertion().containsSubstrings("success", "OK")
+   * }
+ * + * The response assertion will be success when both "success" and "OK" sub strings appear in + * response body (if one or both don't appear, then it fails). You can think of it like {@code + * containsSubstring("success") && containsSubstring("OK")}. + *

+ * If you want to check that any of them matches then use anyMatch, like this: + * + * 

{@code
+   *     responseAssertion().containsSubstrings("success", "OK").anyMatch()
+   * }
+ * + * Which you can interpret as {@code containsSubstring("success") || containsSubstring("OK")}. + * + * @return the Response Assertion to allow configuring other potential settings in a fluent API + */ + public DslResponseAssertion anyMatch() { + this.anyMatch = true; + return this; + } + + @Override + protected TestElement buildTestElement() { + ResponseAssertion ret = new ResponseAssertion(); + setScopeTo(ret); + fieldToTest.applyTo(ret); + ret.setAssumeSuccess(ignoreStatus); + if (invertCheck) { + ret.setToNotType(); + } + if (anyMatch) { + ret.setToOrType(); + } + testStrategy.applyTo(ret); + testStrings.forEach(ret::addTestString); + return ret; + } + + /** + * Identifies a particular field to apply the assertion to. + */ + public enum TargetField { + /** + * Applies the assertion to the response body. + */ + RESPONSE_BODY(ResponseAssertion::setTestFieldResponseData), + /** + * Applies the assertion to the text obtained through Apache + * Tika from the response body (which might be a pdf, excel, etc). + */ + RESPONSE_BODY_AS_DOCUMENT(ResponseAssertion::setTestFieldResponseDataAsDocument), + /** + * Applies the assertion to the response code (eg: the HTTP response code, like 200). + */ + RESPONSE_CODE(ResponseAssertion::setTestFieldResponseCode), + /** + * Applies the assertion to the response message (eg: the HTTP response message, like OK). + */ + RESPONSE_MESSAGE(ResponseAssertion::setTestFieldResponseMessage), + /** + * Applies the assertion to the set of response headers. Response headers is a string with + * headers separated by new lines and names and values separated by colons. + */ + RESPONSE_HEADERS(ResponseAssertion::setTestFieldResponseHeaders), + /** + * Applies the assertion to the set of request headers. Request headers is a string with headers + * separated by new lines and names and values separated by colons. + */ + REQUEST_HEADERS(ResponseAssertion::setTestFieldRequestHeaders), + /** + * Applies the assertion to the requested URL. + */ + REQUEST_URL(ResponseAssertion::setTestFieldURL), + /** + * Applies the assertion to the request body. + */ + REQUEST_BODY(ResponseAssertion::setTestFieldRequestData); + + private final Consumer applier; + + TargetField(Consumer applier) { + this.applier = applier; + } + + private void applyTo(ResponseAssertion assertion) { + applier.accept(assertion); + } + + } + + private enum TestStringStrategy { + CONTAINS_REGEX(ResponseAssertion::setToContainsType), + MATCHES_REGEX(ResponseAssertion::setToMatchType), + SUBSTRING(ResponseAssertion::setToSubstringType), + EQUALS(ResponseAssertion::setToEqualsType); + + private final Consumer applier; + + TestStringStrategy(Consumer applier) { + this.applier = applier; + } + + private void applyTo(ResponseAssertion assertion) { + applier.accept(assertion); + } + + } + +} diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/HtmlReporter.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/HtmlReporter.java index 415ed0b6..98e8c9e2 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/HtmlReporter.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/HtmlReporter.java @@ -16,12 +16,12 @@ import org.apache.jmeter.util.JMeterUtils; import org.apache.jmeter.visualizers.SimpleDataWriter; import us.abstracta.jmeter.javadsl.core.BaseTestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * Generates a nice HTML report at the end of test plan execution. */ -public class HtmlReporter extends BaseTestElement implements MultiScopedTestElement { +public class HtmlReporter extends BaseTestElement implements MultiLevelTestElement { private final File reportDirectory; diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/InfluxDbBackendListener.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/InfluxDbBackendListener.java index 70ad4136..dbd1a913 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/InfluxDbBackendListener.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/InfluxDbBackendListener.java @@ -7,12 +7,12 @@ import org.apache.jmeter.visualizers.backend.BackendListenerGui; import org.apache.jmeter.visualizers.backend.influxdb.InfluxdbBackendListenerClient; import us.abstracta.jmeter.javadsl.core.BaseTestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * Test element which publishes all test run metrics to an InfluxDB instance. */ -public class InfluxDbBackendListener extends BaseTestElement implements MultiScopedTestElement { +public class InfluxDbBackendListener extends BaseTestElement implements MultiLevelTestElement { private final String url; private String title = "Test jmeter-java-dsl " + Instant.now().toString(); diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/JtlWriter.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/JtlWriter.java index 3de6d00a..b1e7e8ee 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/JtlWriter.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/listeners/JtlWriter.java @@ -6,7 +6,7 @@ import org.apache.jmeter.testelement.TestElement; import org.apache.jmeter.visualizers.SimpleDataWriter; import us.abstracta.jmeter.javadsl.core.BaseTestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * Allows to generate a result log file (JTL) with data for each sample for a test plan, thread @@ -23,7 +23,7 @@ * See JMeter listeners doc for * more details on JTL format and settings. */ -public class JtlWriter extends BaseTestElement implements MultiScopedTestElement { +public class JtlWriter extends BaseTestElement implements MultiLevelTestElement { private final String jtlFile; diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslJsr223PostProcessor.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslJsr223PostProcessor.java index 356fd5c2..e3aebad5 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslJsr223PostProcessor.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslJsr223PostProcessor.java @@ -9,7 +9,7 @@ import org.apache.jmeter.util.JSR223TestElement; import org.slf4j.Logger; import us.abstracta.jmeter.javadsl.core.DslJsr223TestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * Allows to run custom logic after getting a sample result. @@ -21,7 +21,7 @@ * for JMeter. If you need, you can use any of JMeter provided scripting languages (beanshell, * javascript, jexl, etc) by setting the {@link #language(String)} property. */ -public class DslJsr223PostProcessor extends DslJsr223TestElement implements MultiScopedTestElement { +public class DslJsr223PostProcessor extends DslJsr223TestElement implements MultiLevelTestElement { private static final String DEFAULT_NAME = "JSR223 PostProcessor"; diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslRegexExtractor.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslRegexExtractor.java index 69081f58..1c3869fc 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslRegexExtractor.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/postprocessors/DslRegexExtractor.java @@ -3,10 +3,9 @@ import java.util.function.Consumer; import org.apache.jmeter.extractor.RegexExtractor; import org.apache.jmeter.extractor.gui.RegexExtractorGui; -import org.apache.jmeter.testelement.AbstractScopedTestElement; import org.apache.jmeter.testelement.TestElement; -import us.abstracta.jmeter.javadsl.core.BaseTestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.DslScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * Allows to extract part of a request or response using regular expressions to store into a @@ -17,7 +16,8 @@ * of parenthesis) of the first match of the regex. If no match is found, then the variable will not * be created or modified. */ -public class DslRegexExtractor extends BaseTestElement implements MultiScopedTestElement { +public class DslRegexExtractor extends DslScopedTestElement implements + MultiLevelTestElement { private final String variableName; private final String regex; @@ -25,8 +25,6 @@ public class DslRegexExtractor extends BaseTestElement implements MultiScopedTes private String template = "$1$"; private String defaultValue; private TargetField fieldToCheck = TargetField.RESPONSE_BODY; - private Scope scope = Scope.MAIN_SAMPLE; - private String scopeVariable; public DslRegexExtractor(String variableName, String regex) { super("name", RegexExtractorGui.class); @@ -42,11 +40,10 @@ public DslRegexExtractor(String variableName, String regex) { * And you use {@code user=([^&]+)} as regular expression. First match (1) would extract {@code * test} and second match (2) would extract {@code tester}. *

- * When not specified, the first match will be used. - * When 0 is specified, a random match will be used. - * When negative, all the matches are extracted, and default value behavior changes. Check JMeter - * Regular Expression Extractor documentation for more details. + * When not specified, the first match will be used. When 0 is specified, a random match will be + * used. When negative, all the matches are extracted to variables with name {@code + * _}, the number of matches is stored in {@code + * _matchNr}, and default value is assigned to {@code }. * * @param matchNumber specifies the match number to use. * @return the DslRegexExtractor to allow fluent usage and setting other properties. @@ -114,41 +111,10 @@ public DslRegexExtractor fieldToCheck(TargetField fieldToCheck) { return this; } - /** - * Allows specifying if the extractor should be specified to main sample and/or sub samples. - *

- * When not specified the regular extractor will only apply to main sample. - * - * @param scope specifying to what sample result apply the regular extractor to. - * @return the DslRegexExtractor to allow fluent usage and setting other properties. - * @see Scope - */ - public DslRegexExtractor scope(Scope scope) { - this.scope = scope; - return this; - } - - /** - * Allows specifying that the regular extractor should be applied to the contents of a given - * JMeter variable. - *

- * This setting overrides any setting on scope and fieldToCheck. - * - * @param scopeVariable specifies the name of the variable to apply the regular extractor to. - * @return the DslRegexExtractor to allow fluent usage and setting other properties. - */ - public DslRegexExtractor scopeVariable(String scopeVariable) { - this.scopeVariable = scopeVariable; - return this; - } - @Override protected TestElement buildTestElement() { RegexExtractor ret = new RegexExtractor(); - scope.applyTo(ret); - if (scopeVariable != null) { - ret.setScopeVariable(scopeVariable); - } + setScopeTo(ret); fieldToCheck.applyTo(ret); ret.setRefName(variableName); ret.setRegex(regex); @@ -182,11 +148,13 @@ public enum TargetField { */ RESPONSE_BODY_AS_DOCUMENT(RegexExtractor::useBodyAsDocument), /** - * Applies the regular extractor to response headers. + * Applies the regular extractor to response headers. Response headers is a string with headers + * separated by new lines and names and values separated by colons. */ RESPONSE_HEADERS(RegexExtractor::useHeaders), /** - * Applies the regular extractor to request headers. + * Applies the regular extractor to request headers. Request headers is a string with headers + * separated by new lines and names and values separated by colons. */ REQUEST_HEADERS(RegexExtractor::useRequestHeaders), /** @@ -213,34 +181,4 @@ private void applyTo(RegexExtractor re) { } } - /** - * Specifies to which samples apply the regular extractor to. - */ - public enum Scope { - /** - * Applies the regular extractor to all samples (main and sub samples). - */ - ALL_SAMPLES(AbstractScopedTestElement::setScopeAll), - /** - * Applies the regular extractor only to main sample (sub samples, like redirects, are not - * included). - */ - MAIN_SAMPLE(AbstractScopedTestElement::setScopeParent), - /** - * Applies the regular extractor only to sub samples (redirects, embedded resources, etc). - */ - SUB_SAMPLES(AbstractScopedTestElement::setScopeChildren); - - private final Consumer applier; - - Scope(Consumer applier) { - this.applier = applier; - } - - private void applyTo(RegexExtractor re) { - applier.accept(re); - } - - } - } diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/preprocessors/DslJsr223PreProcessor.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/preprocessors/DslJsr223PreProcessor.java index 5f4e6e89..f9b21cf2 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/preprocessors/DslJsr223PreProcessor.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/core/preprocessors/DslJsr223PreProcessor.java @@ -8,7 +8,7 @@ import org.apache.jmeter.util.JSR223TestElement; import org.slf4j.Logger; import us.abstracta.jmeter.javadsl.core.DslJsr223TestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * Allows to run custom logic before executing a sampler. @@ -21,7 +21,7 @@ * for JMeter. If you need, you can use any of JMeter provided scripting languages (beanshell, * javascript, jexl, etc) by setting the {@link #language(String)} property. */ -public class DslJsr223PreProcessor extends DslJsr223TestElement implements MultiScopedTestElement { +public class DslJsr223PreProcessor extends DslJsr223TestElement implements MultiLevelTestElement { private static final String DEFAULT_NAME = "JSR223 PreProcessor"; diff --git a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/http/HttpHeaders.java b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/http/HttpHeaders.java index 4940d0c4..ad7b8d3b 100644 --- a/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/http/HttpHeaders.java +++ b/jmeter-java-dsl/src/main/java/us/abstracta/jmeter/javadsl/http/HttpHeaders.java @@ -9,7 +9,7 @@ import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.MimeTypes; import us.abstracta.jmeter.javadsl.core.BaseTestElement; -import us.abstracta.jmeter.javadsl.core.MultiScopedTestElement; +import us.abstracta.jmeter.javadsl.core.MultiLevelTestElement; /** * This class allows specifying HTTP headers (through an underlying JMeter HttpHeaderManager) to be @@ -23,7 +23,7 @@ * that headers specified at lower scope will overwrite ones specified at higher scope (eg: sampler * child headers will overwrite test plan headers). */ -public class HttpHeaders extends BaseTestElement implements MultiScopedTestElement { +public class HttpHeaders extends BaseTestElement implements MultiLevelTestElement { private final Map headers = new HashMap<>(); diff --git a/jmeter-java-dsl/src/test/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertionTest.java b/jmeter-java-dsl/src/test/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertionTest.java new file mode 100644 index 00000000..5063e60b --- /dev/null +++ b/jmeter-java-dsl/src/test/java/us/abstracta/jmeter/javadsl/core/assertions/DslResponseAssertionTest.java @@ -0,0 +1,47 @@ +package us.abstracta.jmeter.javadsl.core.assertions; + +import static org.assertj.core.api.Assertions.assertThat; +import static us.abstracta.jmeter.javadsl.JmeterDsl.responseAssertion; +import static us.abstracta.jmeter.javadsl.JmeterDsl.testPlan; +import static us.abstracta.jmeter.javadsl.JmeterDsl.threadGroup; + +import javax.naming.InitialContext; +import org.junit.jupiter.api.Test; +import us.abstracta.jmeter.javadsl.JmeterDsl; +import us.abstracta.jmeter.javadsl.JmeterDslTest; +import us.abstracta.jmeter.javadsl.core.TestPlanStats; + +public class DslResponseAssertionTest extends JmeterDslTest { + + @Test + public void shouldFailRequestWhenResponseAssertionDoesNotMatch() throws Exception { + TestPlanStats stats = testPlan( + threadGroup(1, 1, + JmeterDsl. + httpSampler(wiremockUri) + .children( + responseAssertion() + .containsSubstrings("test") + ) + ) + ).run(); + assertThat(stats.overall().errorsCount()).isEqualTo(1); + } + + @Test + public void shouldMarkRequestAsSuccessWhenInvalidRequestButResponseAssertionIgnoresStatus() + throws Exception { + TestPlanStats stats = testPlan( + threadGroup(1, 1, + JmeterDsl. + httpSampler("invalidurl") + .children( + responseAssertion() + .ignoreStatus() + ) + ) + ).run(); + assertThat(stats.overall().errorsCount()).isEqualTo(0); + } + +}