diff --git a/CHANGELOG.md b/CHANGELOG.md index 506b6076f..5392a0591 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to AET will be documented in this file. ## Unreleased **List of changes that are finished but not yet released in any final version.** +- [PR-293](https://github.com/Cognifide/aet/pull/293) Added error treshold in pixels and percentages for screen comparator - [PR-300](https://github.com/Cognifide/aet/pull/300) Added creating indexes for collection - [PR-289](https://github.com/Cognifide/aet/pull/289) User now stays on the same tab while navigating between URLs - [PR-271](https://github.com/Cognifide/aet/pull/271) Added possibility to override name parameter from the aet client diff --git a/api/communication-api/src/main/java/com/cognifide/aet/communication/api/metadata/ComparatorStepResult.java b/api/communication-api/src/main/java/com/cognifide/aet/communication/api/metadata/ComparatorStepResult.java index 9c8a39030..210c37dbb 100644 --- a/api/communication-api/src/main/java/com/cognifide/aet/communication/api/metadata/ComparatorStepResult.java +++ b/api/communication-api/src/main/java/com/cognifide/aet/communication/api/metadata/ComparatorStepResult.java @@ -51,6 +51,7 @@ public enum Status { FAILED, WARNING, REBASED, - PROCESSING_ERROR + PROCESSING_ERROR, + CONDITIONALLY_PASSED, } } diff --git a/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparator.java b/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparator.java index 0a9fececf..c20983fd7 100644 --- a/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparator.java +++ b/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparator.java @@ -16,6 +16,8 @@ package com.cognifide.aet.job.common.comparators.layout; import com.cognifide.aet.communication.api.metadata.ComparatorStepResult; +import com.cognifide.aet.communication.api.metadata.ComparatorStepResult.Status; +import com.cognifide.aet.job.api.ParametersValidator; import com.cognifide.aet.job.api.comparator.ComparatorJob; import com.cognifide.aet.job.api.comparator.ComparatorProperties; import com.cognifide.aet.job.api.exceptions.ParametersException; @@ -41,10 +43,18 @@ public class LayoutComparator implements ComparatorJob { public static final String CONTENT_TYPE = "image/png"; + public static final String PERCENTAGE_THRESHOLD_PARAM = "percentageThreshold"; + + public static final String PIXEL_THRESHOLD_PARAM = "pixelThreshold"; + private final ComparatorProperties properties; private final ArtifactsDAO artifactsDAO; + private Integer pixelThreshold; + + private Double percentageThreshold; + LayoutComparator(ComparatorProperties comparatorProperties, ArtifactsDAO artifactsDAO) { this.properties = comparatorProperties; this.artifactsDAO = artifactsDAO; @@ -67,6 +77,7 @@ public ComparatorStepResult compare() throws ProcessingException { BufferedImage collectedImg = ImageIO.read(collectedArtifact); imageComparisonResult = ImageComparison.compare(patternImg, collectedImg); stepResult = saveArtifacts(imageComparisonResult); + } catch (IOException e) { throw new ProcessingException("Error while obtaining artifacts!", e); } @@ -84,7 +95,7 @@ private boolean areInputsIdentical(ArtifactsDAO artifactsDAO, ComparatorProperti private ComparatorStepResult saveArtifacts(ImageComparisonResult imageComparisonResult) throws ProcessingException { final ComparatorStepResult result; - if (imageComparisonResult.isMatch()) { + if (isMaskWithoutDifference(imageComparisonResult)) { result = getPassedStepResult(); } else { InputStream mask = null; @@ -93,19 +104,13 @@ private ComparatorStepResult saveArtifacts(ImageComparisonResult imageComparison mask = new ByteArrayInputStream(baos.toByteArray()); String maskArtifactId = artifactsDAO.saveArtifact(properties, mask, CONTENT_TYPE); - result = new ComparatorStepResult(maskArtifactId, ComparatorStepResult.Status.FAILED, true); - - result.addData("heightDifference", - Integer.toString(imageComparisonResult.getHeightDifference())); - result.addData("widthDifference", - Integer.toString(imageComparisonResult.getWidthDifference())); - result.addData("pixelDifference", - Integer.toString(imageComparisonResult.getPixelDifferenceCount())); - result.addData("patternTimestamp", Long.toString( - artifactsDAO.getArtifactUploadDate(properties, properties.getPatternId()).getTime())); - result.addData("collectTimestamp", Long.toString( - artifactsDAO.getArtifactUploadDate(properties, properties.getCollectedId()) - .getTime())); + if (hasMaskThresholdWithAcceptableDifference(imageComparisonResult)) { + result = new ComparatorStepResult(maskArtifactId, Status.CONDITIONALLY_PASSED, true); + } else { + result = new ComparatorStepResult(maskArtifactId, Status.FAILED, true); + } + addPixelDifferenceDataToResult(result, imageComparisonResult); + addTimestampToResult(result); } catch (Exception e) { throw new ProcessingException(e.getMessage(), e); } finally { @@ -116,18 +121,77 @@ private ComparatorStepResult saveArtifacts(ImageComparisonResult imageComparison return result; } - private ComparatorStepResult getPassedStepResult() { - ComparatorStepResult result = new ComparatorStepResult(null, ComparatorStepResult.Status.PASSED, - false); + boolean hasMaskThresholdWithAcceptableDifference(ImageComparisonResult mask) { + if (pixelThreshold != null && percentageThreshold != null) { + return isAcceptablePixelChange(mask) && this.isAcceptablePercentageChange(mask); + } else if (pixelThreshold != null) { + return isAcceptablePixelChange(mask); + } else if (percentageThreshold != null) { + return isAcceptablePercentageChange(mask); + } + return false; + } + + + public void setPixelThreshold(Integer pixelThreshold) { + this.pixelThreshold = pixelThreshold; + } + + public void setPercentageThreshold(Double percentageThreshold) { + this.percentageThreshold = percentageThreshold; + } + + private boolean isAcceptablePixelChange(ImageComparisonResult mask) { + return mask.getPixelDifferenceCount() <= this.pixelThreshold; + } + + private boolean isAcceptablePercentageChange(ImageComparisonResult mask) { + return mask.getPercentagePixelDifference() <= this.percentageThreshold; + } + + private boolean isMaskWithoutDifference(ImageComparisonResult mask) { + return mask.getHeightDifference() == 0 && mask.getWidthDifference() == 0 + && mask.getPixelDifferenceCount() == 0; + } + + private void addPixelDifferenceDataToResult(ComparatorStepResult result, + ImageComparisonResult imageComparisonResult) { + result.addData("heightDifference", + Integer.toString(imageComparisonResult.getHeightDifference())); + result.addData("widthDifference", + Integer.toString(imageComparisonResult.getWidthDifference())); + result.addData("pixelDifference", + Integer.toString(imageComparisonResult.getPixelDifferenceCount())); + result.addData("percentagePixelDifference", + Double.toString(imageComparisonResult.getPercentagePixelDifference())); + } + + private void addTimestampToResult(ComparatorStepResult result) { result.addData("patternTimestamp", Long.toString( artifactsDAO.getArtifactUploadDate(properties, properties.getPatternId()).getTime())); result.addData("collectTimestamp", Long.toString(System.currentTimeMillis())); + } + + private ComparatorStepResult getPassedStepResult() { + ComparatorStepResult result = new ComparatorStepResult(null, ComparatorStepResult.Status.PASSED, + false); + addTimestampToResult(result); return result; } @Override public void setParameters(Map params) throws ParametersException { - // no parameters needed + if (params.containsKey(PERCENTAGE_THRESHOLD_PARAM)) { + setPercentageThreshold(Double.valueOf(params.get(PERCENTAGE_THRESHOLD_PARAM))); + ParametersValidator + .checkRange(percentageThreshold.intValue(), 0, 100, + "Percentage threshold should be a decimal value between 0 and 100"); + } + if (params.containsKey(PIXEL_THRESHOLD_PARAM)) { + setPixelThreshold(Integer.valueOf(params.get(PIXEL_THRESHOLD_PARAM))); + ParametersValidator.checkParameter(pixelThreshold >= 0, + "Pixel threshold should be greater or equal to 0"); + } } } diff --git a/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonResult.java b/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonResult.java index 878a96805..3c03ff779 100644 --- a/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonResult.java +++ b/core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonResult.java @@ -19,12 +19,16 @@ public class ImageComparisonResult { + private int pixelCount; + private int pixelDifferenceCount; private int heightDifference; private int widthDifference; + private double percentagePixelDifference; + private BufferedImage resultImage; public ImageComparisonResult() { @@ -37,11 +41,8 @@ public ImageComparisonResult(final int pixelDifferenceCount, final int widthDiff this.heightDifference = heightDifference; this.widthDifference = widthDifference; this.resultImage = resultImage; - } - - public boolean isMatch() { - return this.pixelDifferenceCount == 0 && this.heightDifference == 0 - && this.widthDifference == 0; + this.pixelCount = resultImage.getHeight() * resultImage.getWidth(); + this.percentagePixelDifference = 100 * this.pixelDifferenceCount / (double) this.pixelCount; } public int getPixelDifferenceCount() { @@ -59,4 +60,9 @@ public int getWidthDifference() { public BufferedImage getResultImage() { return resultImage; } + + public double getPercentagePixelDifference() { + return percentagePixelDifference; + } + } diff --git a/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparatorTest.java b/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparatorTest.java new file mode 100644 index 000000000..4a35f0322 --- /dev/null +++ b/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparatorTest.java @@ -0,0 +1,143 @@ +/** + * AET + * + * Copyright (C) 2013 Cognifide Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.cognifide.aet.job.common.comparators.layout; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.when; + +import com.cognifide.aet.job.api.comparator.ComparatorProperties; +import com.cognifide.aet.job.common.comparators.layout.utils.ImageComparisonResult; +import com.cognifide.aet.vs.ArtifactsDAO; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class LayoutComparatorTest { + + @Mock + private ComparatorProperties comparatorProperties; + + @Mock + private ArtifactsDAO artifactsDAO; + + @Mock + private ImageComparisonResult imageComparisonResult; + + private LayoutComparator layoutComparator; + + @Before + public void setUp() { + //given + this.layoutComparator = new LayoutComparator(this.comparatorProperties, this.artifactsDAO); + } + + @Test + public void hasMaskThresholdWithAcceptableDifference_withoutThreshold_expectFalse() { + //when + when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567); + when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300); + + //then + assertThat( + this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(false)); + } + + @Test + public void hasMaskThresholdWithAcceptableDifference_withThreshold_expectFalse() { + //when + when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567); + when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300); + + this.layoutComparator.setPixelThreshold(299); + this.layoutComparator.setPercentageThreshold(null); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(false)); + + //when + this.layoutComparator.setPixelThreshold(null); + this.layoutComparator.setPercentageThreshold(12.566); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(false)); + } + + @Test + public void hasMaskThresholdWithAcceptableDifference_withThreshold_expectTrue() { + //when + when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567); + when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300); + + this.layoutComparator.setPixelThreshold(300); + this.layoutComparator.setPercentageThreshold(null); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(true)); + + //when + this.layoutComparator.setPixelThreshold(null); + this.layoutComparator.setPercentageThreshold(12.567); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(true)); + } + + @Test + public void hasMaskThresholdWithAcceptableDifference_withBothThreshold_expectFalse() { + //when + when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567); + when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300); + + this.layoutComparator.setPixelThreshold(299); + this.layoutComparator.setPercentageThreshold(30.0); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(false)); + + //when + this.layoutComparator.setPixelThreshold(301); + this.layoutComparator.setPercentageThreshold(12.566); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(false)); + } + + @Test + public void hasMaskThresholdWithAcceptableDifference_withBothThreshold_expectTrue() { + //when + when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567); + when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300); + + this.layoutComparator.setPixelThreshold(300); + this.layoutComparator.setPercentageThreshold(12.567); + + //then + assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult), + is(true)); + } + +} diff --git a/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonTest.java b/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonTest.java index a96e6d575..57514e978 100644 --- a/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonTest.java +++ b/core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonTest.java @@ -45,7 +45,7 @@ public void test_sameScreenshot_expectNoDifferencesInResultAndTransparentMask() // when ImageComparisonResult imageComparisonResult = ImageComparison.compare(pattern, sample); // then - assertThat(imageComparisonResult.isMatch(), is(true)); + assertThat(imageComparisonResult.getPercentagePixelDifference(), is(0.0)); assertThat(imageComparisonResult.getHeightDifference(), is(0)); assertThat(imageComparisonResult.getWidthDifference(), is(0)); assertThat(imageComparisonResult.getPixelDifferenceCount(), is(0)); @@ -76,7 +76,7 @@ public void testCompare_different() throws Exception { // when ImageComparisonResult imageComparisonResult = ImageComparison.compare(pattern, sample); // then - assertThat(imageComparisonResult.isMatch(), is(false)); + assertThat(imageComparisonResult.getPercentagePixelDifference(), is(0.7435369480668039)); assertThat(imageComparisonResult.getHeightDifference(), is(0)); assertThat(imageComparisonResult.getWidthDifference(), is(0)); assertThat(imageComparisonResult.getPixelDifferenceCount(), is(15600)); @@ -110,7 +110,7 @@ public void compare_differentSizeScreenshots_expectSizeDifferenceMarkedWithYello // when ImageComparisonResult imageComparisonResult = ImageComparison.compare(pattern, sample); // then - assertThat(imageComparisonResult.isMatch(), is(false)); + assertThat(imageComparisonResult.getPercentagePixelDifference(), is(59.99583333333333)); assertThat(imageComparisonResult.getHeightDifference(), is(100)); assertThat(imageComparisonResult.getWidthDifference(), is(20)); assertThat(imageComparisonResult.getPixelDifferenceCount(), is(14399)); diff --git a/documentation/src/main/wiki/LayoutComparator.md b/documentation/src/main/wiki/LayoutComparator.md index 2d044119a..16d681973 100644 --- a/documentation/src/main/wiki/LayoutComparator.md +++ b/documentation/src/main/wiki/LayoutComparator.md @@ -10,7 +10,12 @@ Resource name: screen ##### Parameters -No parameters +| Parameter | Value | Description | Mandatory | +| --------- | ----- | ----------- | --------- | +| `pixelThreshold` | int (equal or greater than 0) | The value to set the error threshold in pixels e.g if difference between photos is smaller or equal to `pixelThreshold`, the test will pass. In case of difference is bigger than `pixelThreshold`, the test will fail. | no | +| `percentageThreshold` | double (between 0 and 100) | It works as `pixelThreshold` but values are in percentages | no | + +When you provide `pixelThreshold` and `percentageThreshold` test will pass only if pixel difference is smaller or equal than `pixelThreshold` and percentage difference is smaller or equal than `percentageThreshold`. ##### Example Usage @@ -25,7 +30,7 @@ No parameters ... - + ... diff --git a/documentation/src/main/wiki/SuiteReportFeatures.md b/documentation/src/main/wiki/SuiteReportFeatures.md index 21db93b4f..5f3eeb322 100644 --- a/documentation/src/main/wiki/SuiteReportFeatures.md +++ b/documentation/src/main/wiki/SuiteReportFeatures.md @@ -21,9 +21,9 @@ Every test case has one of the following states: 1. passed — if the comparator doesn't find any change, i.e. validation passes, 2. passed with warnings — if there are some warnings, but they are not very important, -3. failed — if the comparator detects some changes or some validation rules are broken, -4. accepted — if failed test was accepted. -The state of the test case is propagated to the URL then to the test and to the test suite. It is possible to accept all test cases for a given URL or in a current test suite. +3. conditionally passed - if the comparator finds some change, but the change is smaller or equal to error threshold +4. failed — if the comparator detects some changes or some validation rules are broken, +5. accepted — if failed test was accepted. Tests and URLs may be filtered by: diff --git a/documentation/src/main/wiki/SuiteReportLayoutCase.md b/documentation/src/main/wiki/SuiteReportLayoutCase.md index f2f6e2164..8217d062f 100644 --- a/documentation/src/main/wiki/SuiteReportLayoutCase.md +++ b/documentation/src/main/wiki/SuiteReportLayoutCase.md @@ -32,6 +32,12 @@ Test case's result is marked as successful when there is no difference between v ![Layout success](assets/suiteReport/layout-success.png) +##### Conditionally passed + +Test case's result is marked as conditionally passed when there is difference between view and pattern (see screenshot below) but the difference is below threshold - here `percentageThreshold="5"`. The test is pass so you can't accept it, therefore "Accept test case" button isn't available and the test has green color (but with different icon to be perceptible) but images are different so you can see a mask. + +![Layout conditionally passed](assets/suiteReport/layout-conditionally-passed.png) + ##### What vulnerabilities it discovers * Differences found in page screenshots may indicate undesired changes in the page layout (css, html structure) e.g. when a new functionality was implemented in a system it may have an impact on another system component(s). This may show itself as a changed page layout. diff --git a/documentation/src/main/wiki/assets/suiteReport/layout-conditionally-passed.PNG b/documentation/src/main/wiki/assets/suiteReport/layout-conditionally-passed.PNG new file mode 100644 index 000000000..df1c55ebb Binary files /dev/null and b/documentation/src/main/wiki/assets/suiteReport/layout-conditionally-passed.PNG differ diff --git a/integration-tests/sanity-functional/src/main/java/com/cognifide/aet/sanity/functional/TestStatus.java b/integration-tests/sanity-functional/src/main/java/com/cognifide/aet/sanity/functional/TestStatus.java index cfe407183..40c1bf217 100644 --- a/integration-tests/sanity-functional/src/main/java/com/cognifide/aet/sanity/functional/TestStatus.java +++ b/integration-tests/sanity-functional/src/main/java/com/cognifide/aet/sanity/functional/TestStatus.java @@ -18,6 +18,7 @@ public enum TestStatus { SUCCESS("passed"), + CONDITIONALLY_PASSED("conditionallyPassed"), WARN("warning"), FAIL("failed"), REBASED("rebased"); diff --git a/integration-tests/sanity-functional/src/test/java/com/cognifide/aet/sanity/functional/HomePageTilesTest.java b/integration-tests/sanity-functional/src/test/java/com/cognifide/aet/sanity/functional/HomePageTilesTest.java index 3d41f365c..1644a1a86 100644 --- a/integration-tests/sanity-functional/src/test/java/com/cognifide/aet/sanity/functional/HomePageTilesTest.java +++ b/integration-tests/sanity-functional/src/test/java/com/cognifide/aet/sanity/functional/HomePageTilesTest.java @@ -29,13 +29,15 @@ @Modules(GuiceModule.class) public class HomePageTilesTest { - private static final int TESTS = 114; + private static final int TESTS = 122; - private static final int EXPECTED_TESTS_SUCCESS = 66; + private static final int EXPECTED_TESTS_SUCCESS = 69; + + private static final int EXPECTED_TESTS_CONDITIONALLY_PASSED = 3; private static final int EXPECTED_TESTS_WARN = 5; - private static final int EXPECTED_TESTS_FAIL = 43; + private static final int EXPECTED_TESTS_FAIL = 48; @Inject private ReportHomePage page; @@ -55,7 +57,17 @@ public void checkNumberOfTiles() { public void checkNumberOfSuccessTiles() { String cssClassToSearch = TestStatus.SUCCESS.getCssClass(); int tiles = page.findTiles(cssClassToSearch).size(); - assertEquals("number of tests tiles with SUCCESS status is incorrect", EXPECTED_TESTS_SUCCESS, + assertEquals("number of tests tiles with SUCCESS status is incorrect", + EXPECTED_TESTS_SUCCESS - EXPECTED_TESTS_CONDITIONALLY_PASSED, + tiles); + } + + @Test + public void checkNumberOfConditionallyPassedTiles() { + String cssClassToSearch = TestStatus.CONDITIONALLY_PASSED.getCssClass(); + int tiles = page.findTiles(cssClassToSearch).size(); + assertEquals("number of tests tiles with CONDITIONALLY PASSED status is incorrect", + EXPECTED_TESTS_CONDITIONALLY_PASSED, tiles); } diff --git a/integration-tests/sanity-functional/src/test/resources/features/filtering.feature b/integration-tests/sanity-functional/src/test/resources/features/filtering.feature index 9b4143318..1f17aa3b4 100644 --- a/integration-tests/sanity-functional/src/test/resources/features/filtering.feature +++ b/integration-tests/sanity-functional/src/test/resources/features/filtering.feature @@ -26,76 +26,76 @@ Feature: Tests Results Filtering Given I have opened sample tests report page When I search for tests containing "accessibility" Then There are 16 tiles visible - And Statistics text contains "16 ( 8 / 3 / 5 / 0 )" + And Statistics text contains "16 ( 8 / 3 / 5 (0) / 0 )" Scenario: Filtering Tests Results: cookie Given I have opened sample tests report page When I search for tests containing "cookie" Then There are 9 tiles visible - And Statistics text contains "9 ( 4 / 0 / 5 / 0 )" + And Statistics text contains "9 ( 4 / 0 / 5 (0) / 0 )" Scenario: Filtering Tests Results: layout Given I have opened sample tests report page When I search for tests containing "layout" - Then There are 13 tiles visible - And Statistics text contains "13 ( 3 / 0 / 10 / 0 )" + Then There are 21 tiles visible + And Statistics text contains "21 ( 8 / 0 / 13 (3) / 0 )" Scenario: Filtering Tests Results: jserrors Given I have opened sample tests report page When I search for tests containing "jserrors" Then There are 14 tiles visible - And Statistics text contains "14 ( 6 / 0 / 8 / 0 )" + And Statistics text contains "14 ( 6 / 0 / 8 (0) / 0 )" Scenario: Filtering Tests Results: source Given I have opened sample tests report page When I search for tests containing "source" Then There are 17 tiles visible - And Statistics text contains "17 ( 7 / 0 / 10 / 0 )" + And Statistics text contains "17 ( 7 / 0 / 10 (0) / 0 )" Scenario: Filtering Tests Results: status-codes Given I have opened sample tests report page When I search for tests containing "status" Then There are 21 tiles visible - And Statistics text contains "21 ( 9 / 0 / 12 / 0 )" + And Statistics text contains "21 ( 9 / 0 / 12 (0) / 0 )" Scenario: Filtering Tests Results: w3c-html5 Given I have opened sample tests report page When I search for tests containing "w3c-html5" Then There are 8 tiles visible - And Statistics text contains "8 ( 1 / 1 / 6 / 0 )" + And Statistics text contains "8 ( 1 / 1 / 6 (0) / 0 )" Scenario: Filtering Tests Results: click-modifier Given I have opened sample tests report page When I search for tests containing "click" Then There are 1 tiles visible - And Statistics text contains "1 ( 0 / 0 / 1 / 0 )" + And Statistics text contains "1 ( 0 / 0 / 1 (0) / 0 )" Scenario: Filtering Tests Results: header Given I have opened sample tests report page When I search for tests containing "header" Then There are 1 tiles visible - And Statistics text contains "1 ( 0 / 0 / 1 / 0 )" + And Statistics text contains "1 ( 0 / 0 / 1 (0) / 0 )" Scenario: Filtering Tests Results: sleep-modifier Given I have opened sample tests report page When I search for tests containing "sleep" Then There are 2 tiles visible - And Statistics text contains "2 ( 1 / 0 / 1 / 0 )" + And Statistics text contains "2 ( 1 / 0 / 1 (0) / 0 )" Scenario: Filtering Tests Results: wait-for-page-modifier Given I have opened sample tests report page When I search for tests containing "wait-for-page-loaded" Then There are 2 tiles visible - And Statistics text contains "2 ( 1 / 0 / 1 / 0 )" + And Statistics text contains "2 ( 1 / 0 / 1 (0) / 0 )" Scenario: Filtering Tests Results: wait-for-element-to-be-visible-modifier Given I have opened sample tests report page When I search for tests containing "wait-for-element-to-be-visible" Then There are 2 tiles visible - And Statistics text contains "2 ( 1 / 0 / 1 / 0 )" + And Statistics text contains "2 ( 1 / 0 / 1 (0) / 0 )" Scenario: Filtering Tests Results: wait-for-image-completion-modifier Given I have opened sample tests report page When I search for tests containing "wait-for-image-completion" Then There are 4 tiles visible - And Statistics text contains "4 ( 2 / 0 / 2 / 0 )" + And Statistics text contains "4 ( 2 / 0 / 2 (0) / 0 )" diff --git a/integration-tests/test-suite/partials/layout.xml b/integration-tests/test-suite/partials/layout.xml index 913c61447..1e10446f8 100644 --- a/integration-tests/test-suite/partials/layout.xml +++ b/integration-tests/test-suite/partials/layout.xml @@ -148,5 +148,117 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/report/src/main/webapp/app/app.module.js b/report/src/main/webapp/app/app.module.js index e5bf9d669..30c6e70af 100644 --- a/report/src/main/webapp/app/app.module.js +++ b/report/src/main/webapp/app/app.module.js @@ -88,7 +88,8 @@ define(['angularAMD', failed: 'fa-times', warning: 'fa-exclamation-triangle', rebased: 'fa-cloud-upload-alt', - unrebased: 'fa-cloud-download-alt' + unrebased: 'fa-cloud-download-alt', + conditionallyPassed: 'fa-dot-circle', } }; diff --git a/report/src/main/webapp/app/layout/main/suite/mainView.suite.view.html b/report/src/main/webapp/app/layout/main/suite/mainView.suite.view.html index 0d08dd937..713c12b7a 100644 --- a/report/src/main/webapp/app/layout/main/suite/mainView.suite.view.html +++ b/report/src/main/webapp/app/layout/main/suite/mainView.suite.view.html @@ -26,10 +26,10 @@
-
+
-
+
{{test.name}} diff --git a/report/src/main/webapp/app/layout/main/test/mainView.test.view.html b/report/src/main/webapp/app/layout/main/test/mainView.test.view.html index 0400c9c29..c28488b96 100644 --- a/report/src/main/webapp/app/layout/main/test/mainView.test.view.html +++ b/report/src/main/webapp/app/layout/main/test/mainView.test.view.html @@ -27,10 +27,10 @@
-
+
-
+
@@ -41,6 +41,8 @@ data-trigger="hover" data-container="body">{{url.warning}} / {{url.passed}} / + {{url.conditionallyPassed}} / {{url.rebased}}
diff --git a/report/src/main/webapp/app/layout/main/url/reports/screen_layout.html b/report/src/main/webapp/app/layout/main/url/reports/screen_layout.html index ad35c5791..7e4fabb7d 100644 --- a/report/src/main/webapp/app/layout/main/url/reports/screen_layout.html +++ b/report/src/main/webapp/app/layout/main/url/reports/screen_layout.html @@ -40,7 +40,8 @@

Pattern - {{case.comparator.stepResult.data.patternTimestamp | date:'yyyy-MM-dd HH:mm'}} + {{case.comparator.stepResult.data.patternTimestamp | date:'yyyy-MM-dd HH:mm'}}
+

-
+

Collected - {{case.comparator.stepResult.data.collectTimestamp | date:'yyyy-MM-dd HH:mm'}} + {{case.comparator.stepResult.data.collectTimestamp | date:'yyyy-MM-dd HH:mm'}}
+ Difference: {{case.comparator.stepResult.data.percentagePixelDifference | number: 2}}%, + {{case.comparator.stepResult.data.pixelDifference}}px

@@ -68,7 +71,7 @@
-
+

No difference was detected!

diff --git a/report/src/main/webapp/app/layout/sidepanel/sidepanel.controller.js b/report/src/main/webapp/app/layout/sidepanel/sidepanel.controller.js index 73585b1e3..4d9912d9b 100644 --- a/report/src/main/webapp/app/layout/sidepanel/sidepanel.controller.js +++ b/report/src/main/webapp/app/layout/sidepanel/sidepanel.controller.js @@ -68,6 +68,7 @@ define([], function () { function updateNavigationTree() { vm.testsStats = { total: 0, + conditionallyPassed: 0, failed: 0, warning: 0, passed: 0, @@ -83,6 +84,7 @@ define([], function () { refreshTestsStatsValue('warning'); refreshTestsStatsValue('passed'); refreshTestsStatsValue('rebased'); + refreshTestsStatsValue('conditionallyPassed'); $scope.$apply(); } diff --git a/report/src/main/webapp/app/layout/sidepanel/sidepanel.view.html b/report/src/main/webapp/app/layout/sidepanel/sidepanel.view.html index 169cd1758..40604fbe0 100644 --- a/report/src/main/webapp/app/layout/sidepanel/sidepanel.view.html +++ b/report/src/main/webapp/app/layout/sidepanel/sidepanel.view.html @@ -82,6 +82,11 @@

Show passed urls

+
+ +
@@ -124,6 +131,7 @@ type="test-name" ng-class="[test.getStatus()]" data-url="{{test.name}}"> + {{test.name}}
-
- -
-
- -
+
+ +
+
+ +
{{url.name}} diff --git a/report/src/main/webapp/app/layout/toolbar/toolbarTop.view.html b/report/src/main/webapp/app/layout/toolbar/toolbarTop.view.html index b05efa260..d91edb569 100644 --- a/report/src/main/webapp/app/layout/toolbar/toolbarTop.view.html +++ b/report/src/main/webapp/app/layout/toolbar/toolbarTop.view.html @@ -49,7 +49,12 @@ data-trigger="hover" data-container="body">{{toolbarTop.suiteStatistics.warning}} / {{toolbarTop.suiteStatistics.passed}} + data-trigger="hover" data-container="body"> + {{toolbarTop.suiteStatistics.passed}} + + ({{toolbarTop.suiteStatistics.conditionallyPassed}}) + / {{toolbarTop.suiteStatistics.rebased}} ) diff --git a/report/src/main/webapp/app/services/metadata.service.js b/report/src/main/webapp/app/services/metadata.service.js index d52869fed..9e7f10645 100644 --- a/report/src/main/webapp/app/services/metadata.service.js +++ b/report/src/main/webapp/app/services/metadata.service.js @@ -163,6 +163,7 @@ define(['angularAMD', 'metadataCacheService', 'metadataEndpointService'], failed: failed, warning: warning, rebased: rebased, + conditionallyPassed: conditionallyPassed, updateStatistics: updateStatistics, updatePatternStatistics: updatePatternStatistics, revertPatternStatistics: revertPatternStatistics, @@ -180,6 +181,7 @@ define(['angularAMD', 'metadataCacheService', 'metadataEndpointService'], decoratedObject.warning = 0; decoratedObject.passed = 0; decoratedObject.rebased = 0; + decoratedObject.conditionallyPassed = 0; decoratedObject.patternsToAccept = 0; decoratedObject.acceptedPatterns = 0; decoratedObject.getStatus = getStatus; @@ -195,6 +197,8 @@ define(['angularAMD', 'metadataCacheService', 'metadataEndpointService'], status = 'rebased'; } else if (decoratedObject.warning > 0) { status = 'warning'; + } else if (decoratedObject.conditionallyPassed > 0) { + status = 'conditionallyPassed'; } return status; } @@ -287,6 +291,14 @@ define(['angularAMD', 'metadataCacheService', 'metadataEndpointService'], } } + function conditionallyPassed() { + decoratedObject.total++; + decoratedObject.conditionallyPassed++; + if (decoratedParentReference && decoratedParentReference.conditionallyPassed) { + decoratedParentReference.conditionallyPassed(); + } + } + function updatePatternsToAccept(numberOfPatternsToAccept) { decoratedObject.patternsToAccept += numberOfPatternsToAccept; if (decoratedParentReference && @@ -311,6 +323,9 @@ define(['angularAMD', 'metadataCacheService', 'metadataEndpointService'], case 'PASSED': passed(); break; + case 'CONDITIONALLY_PASSED': + conditionallyPassed(); + break; case 'FAILED': // if `rebaseable` comparator, increment acceptable patterns counter if (stepResult.rebaseable) { diff --git a/report/src/main/webapp/assets/sass/_report.scss b/report/src/main/webapp/assets/sass/_report.scss index 6542bccbd..8af5d7957 100644 --- a/report/src/main/webapp/assets/sass/_report.scss +++ b/report/src/main/webapp/assets/sass/_report.scss @@ -114,6 +114,15 @@ } } + &.conditionallyPassed { + background: $passed_bg; + border-top: solid 10px $passed; + + span, svg { + color: $conditionallyPassed; + } + } + &:hover { .test-icons { z-index: 2; @@ -288,6 +297,13 @@ ul { color: $warning; } } + li.conditionallyPassed { + a.nav-link, + a.nav-link:focus, + a.nav-link:hover { + color: $conditionallyPassed; + } + } } // cookies report diff --git a/report/src/main/webapp/assets/sass/_sidebar.scss b/report/src/main/webapp/assets/sass/_sidebar.scss index eb9564cad..202027816 100644 --- a/report/src/main/webapp/assets/sass/_sidebar.scss +++ b/report/src/main/webapp/assets/sass/_sidebar.scss @@ -104,6 +104,10 @@ color: $rebased; } + &.conditionallyPassed { + color: $conditionallyPassed; + } + &.is-hidden { + .urls-list { display: none; @@ -193,6 +197,10 @@ &.rebased { color: $rebased; } + + &.conditionallyPassed { + color: $conditionallyPassed; + } } } } diff --git a/report/src/main/webapp/assets/sass/_test.scss b/report/src/main/webapp/assets/sass/_test.scss index 7c47c17d9..502698717 100644 --- a/report/src/main/webapp/assets/sass/_test.scss +++ b/report/src/main/webapp/assets/sass/_test.scss @@ -175,6 +175,10 @@ } } + .label-spacer:before { + content: "\200b"; // unicode zero width space character + } + } &-description { diff --git a/report/src/main/webapp/assets/sass/_toolbar.scss b/report/src/main/webapp/assets/sass/_toolbar.scss index 5b808504d..8a709beb5 100644 --- a/report/src/main/webapp/assets/sass/_toolbar.scss +++ b/report/src/main/webapp/assets/sass/_toolbar.scss @@ -51,7 +51,7 @@ &-link { @include transition(width, 0.3s); - width: calc(100% - 400px); + width: calc(100% - 450px); background: $white; > { @@ -99,6 +99,26 @@ } } + &-generate-raport { + + display:flex; + align-items: center; + justify-content: center; + + .raport-button { + background-color: $rebased_bg; + padding:10px 15px; + font-weight: bold; + color:$text_color; + + &:hover { + cursor:pointer; + } + } + + + } + &-bottom { display: flex; justify-content: space-between; diff --git a/report/src/main/webapp/assets/sass/_variables.scss b/report/src/main/webapp/assets/sass/_variables.scss index 1c3fe7c38..2b240fbf2 100644 --- a/report/src/main/webapp/assets/sass/_variables.scss +++ b/report/src/main/webapp/assets/sass/_variables.scss @@ -60,6 +60,7 @@ $warning: #f0ad4e; $rebased: #0097fe; $failed: #bb5a5a; $passed: #6f9f00; +$conditionallyPassed: #6f9f00; $warning_bg: #fff085; $rebased_bg: #00c1ff; diff --git a/report/src/main/webapp/assets/sass/main.scss b/report/src/main/webapp/assets/sass/main.scss index ec078732b..ded719fcb 100644 --- a/report/src/main/webapp/assets/sass/main.scss +++ b/report/src/main/webapp/assets/sass/main.scss @@ -154,6 +154,10 @@ body { &.passed { color: $passed !important; } + + &.conditionallyPassed { + color: $conditionallyPassed !important; + } } .url-tile { @@ -179,6 +183,10 @@ body { color: $failed; } + &.conditionallyPassed { + color: $conditionallyPassed; + } + .statistics { color: $gray_dark !important;; }