Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[TEST] Refactor the computing of the snapshot & diff paths for the visual tests #939

Merged
merged 4 commits into from
Dec 14, 2020
Merged

[TEST] Refactor the computing of the snapshot & diff paths for the visual tests #939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b2af23b
Move custom diff & snapshots directories in ImageSnapshotConfigurator
csouchet Dec 8, 2020
2f625aa
Refactor calculation of custom dirs in Diagram rendering visual tests
csouchet Dec 9, 2020
c8cc1a3
Merge branch 'master' into 903-Organize_screenshots_of_visual_tests
csouchet Dec 14, 2020
33f8bf4
Merge branch 'master' into 903-Organize_screenshots_of_visual_tests
csouchet Dec 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions test/e2e/bpmn.navigation.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,9 @@ describe('diagram navigation', () => {
},
],
]),
'navigation',
);

const navigationDiffDir = join(ImageSnapshotConfigurator.getDiffDir(), 'navigation');
const navigationDir = join(ImageSnapshotConfigurator.getSnapshotsDir(), 'navigation');

// to have mouse pointer visible during headless test - add 'showMousePointer=true' to queryParams
const bpmnDiagramPreparation = new BpmnDiagramPreparation(new Map(), { name: 'rendering-diagram', queryParams: [] }, 'navigation');

Expand Down Expand Up @@ -65,8 +63,6 @@ describe('diagram navigation', () => {
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: 'mouse.panning',
customSnapshotsDir: navigationDir,
customDiffDir: navigationDiffDir,
});
});

Expand All @@ -83,8 +79,6 @@ describe('diagram navigation', () => {
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: zoom === 'zoom in' ? 'mouse.zoom.in' : 'mouse.zoom.out',
customSnapshotsDir: navigationDir,
customDiffDir: navigationDiffDir,
});
});

Expand All @@ -111,8 +105,7 @@ describe('diagram navigation', () => {
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: 'initial.zoom',
customSnapshotsDir: navigationDir,
customDiffDir: join(navigationDiffDir, `${xTimes}-zoom-in-out`),
customDiffDir: join(config.customDiffDir, `${xTimes}-zoom-in-out`),
});
});
});
12 changes: 2 additions & 10 deletions test/e2e/bpmn.rendering.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/
import { findFiles } from '../helpers/file-helper';
import { BpmnDiagramPreparation, BpmnLoadMethod, ImageSnapshotConfigurator, ImageSnapshotThresholdConfig, PageTester } from './helpers/visu-utils';
import { join } from 'path';

describe('no BPMN elements visual regression', () => {
const imageSnapshotConfigurator = new ImageSnapshotConfigurator(
Expand Down Expand Up @@ -109,6 +108,7 @@ describe('no BPMN elements visual regression', () => {
},
],
]),
'bpmn',
);

const bpmnDiagramPreparation = new BpmnDiagramPreparation(
Expand All @@ -130,9 +130,6 @@ describe('no BPMN elements visual regression', () => {
return filename.split('.').slice(0, -1).join('.');
});

const bpmnDiffDir = join(ImageSnapshotConfigurator.getDiffDir(), 'bpmn');
const bpmnDir = join(ImageSnapshotConfigurator.getSnapshotsDir(), 'bpmn');

it('check bpmn non-regression files availability', () => {
expect(bpmnFileNames).toContain('gateways');
});
Expand All @@ -142,11 +139,6 @@ describe('no BPMN elements visual regression', () => {

const image = await page.screenshot({ fullPage: true });
const config = imageSnapshotConfigurator.getConfig(fileName);
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: fileName,
customSnapshotsDir: bpmnDir,
customDiffDir: bpmnDiffDir,
});
expect(image).toMatchImageSnapshot(config);
});
});
110 changes: 65 additions & 45 deletions test/e2e/diagram.rendering.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,40 @@
import { BpmnDiagramPreparation, BpmnLoadMethod, ImageSnapshotConfigurator, ImageSnapshotThresholdConfig, PageTester } from './helpers/visu-utils';
import { FitType, LoadOptions } from '../../src/component/options';
import { join } from 'path';
import { MatchImageSnapshotOptions } from 'jest-image-snapshot';

function getCustomSnapshotDir(fitType: FitType, margin = 0): string {
const fitDir = join(ImageSnapshotConfigurator.getSnapshotsDir(), 'fit');
const typeDir = join(fitDir, `type-${fitType}`);
return join(typeDir, `margin-${margin == null || margin < 0 ? 0 : margin}`);
class FitImageSnapshotConfigurator extends ImageSnapshotConfigurator {
getConfig(param: {
fileName: string;
buildCustomDiffDir: (config: MatchImageSnapshotOptions, fitType: FitType, margin?: number) => string;
fitType: FitType;
margin?: number;
}): MatchImageSnapshotOptions {
const config = super.getConfig(param);
config.customSnapshotsDir = FitImageSnapshotConfigurator.buildSnapshotFitDir(config.customSnapshotsDir, param.fitType, true, param.margin ? param.margin : 0);
config.customDiffDir = param.buildCustomDiffDir(config, param.fitType, param.margin);
return config;
}

private static buildSnapshotFitDir(parentDir: string, fitType: FitType, withMargin = false, margin?: number): string {
const typeDir = join(parentDir, `type-${fitType}`);

if (!withMargin) {
return typeDir;
}
return join(typeDir, `margin-${margin == null || margin < 0 ? 0 : margin}`);
}

static buildOnLoadDiffDir(config: MatchImageSnapshotOptions, fitType: FitType, withMargin = false, margin?: number): string {
const onLoadDir = join(config.customDiffDir, 'on-load');
return FitImageSnapshotConfigurator.buildSnapshotFitDir(onLoadDir, fitType, withMargin, margin);
}

static buildAfterLoadDiffDir(config: MatchImageSnapshotOptions, afterLoadFitType: FitType, onLoadFitType: FitType): string {
const afterLoadDir = join(config.customDiffDir, 'after-load');
const snapshotFitTypeDir = FitImageSnapshotConfigurator.buildSnapshotFitDir(afterLoadDir, afterLoadFitType);
return join(snapshotFitTypeDir, `on-load_type-${onLoadFitType}`);
}
}

/* eslint-disable @typescript-eslint/no-explicit-any */
Expand All @@ -30,10 +59,8 @@ async function initializePage(loadOptions: LoadOptions, fileName: string): Promi
await pageTester.expectBpmnDiagramToBeDisplayed(fileName);
}

const loadDiffDir = join(ImageSnapshotConfigurator.getDiffDir(), 'load');

describe('no diagram visual regression', () => {
const imageSnapshotConfigurator = new ImageSnapshotConfigurator(
const imageSnapshotConfigurator = new FitImageSnapshotConfigurator(
new Map<string, ImageSnapshotThresholdConfig>([
[
'with.outside.labels',
Expand All @@ -48,71 +75,64 @@ describe('no diagram visual regression', () => {
},
],
]),
'fit',
// minimal threshold to make test pass on Github Workflow
// ubuntu: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
// macOS: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
// windows: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
0.00006,
);

const fitTypes: FitType[] = [FitType.None, FitType.HorizontalVertical, FitType.Horizontal, FitType.Vertical, FitType.Center];
describe.each(fitTypes)('load options - fit %s', (loadFitType: FitType) => {
const loadFitDiffDir = join(loadDiffDir, `type-${loadFitType}`);
const fitDiffDir = join(loadFitDiffDir, 'fit');

describe.each(fitTypes)('load options - fit %s', (onLoadFitType: FitType) => {
describe.each(['horizontal', 'vertical', 'with.outside.flows', 'with.outside.labels'])('diagram %s', (fileName: string) => {
it('load', async () => {
await initializePage({ fit: { type: loadFitType } }, fileName);
await initializePage({ fit: { type: onLoadFitType } }, fileName);

const image = await page.screenshot({ fullPage: true });

// minimal threshold to make test pass on Github Workflow
// ubuntu: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
// macOS: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
// windows: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
const config = imageSnapshotConfigurator.getConfig(fileName, 0.00006);
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: fileName,
customSnapshotsDir: getCustomSnapshotDir(loadFitType),
customDiffDir: loadFitDiffDir,
const config = imageSnapshotConfigurator.getConfig({
fileName,
fitType: onLoadFitType,
buildCustomDiffDir: (config, fitType) => FitImageSnapshotConfigurator.buildOnLoadDiffDir(config, fitType),
});
expect(image).toMatchImageSnapshot(config);
});

it.each(fitTypes)(`load + fit %s`, async (fitType: FitType) => {
await initializePage({ fit: { type: loadFitType } }, fileName);
it.each(fitTypes)(`load + fit %s`, async (afterLoadFitType: FitType) => {
await initializePage({ fit: { type: onLoadFitType } }, fileName);

await page.click(`#${fitType}`);
await page.click(`#${afterLoadFitType}`);
// To unselect the button
await page.mouse.click(0, 0);

const image = await page.screenshot({ fullPage: true });

// minimal threshold to make test pass on Github Workflow
// ubuntu: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
// macos: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
// windows: Expected image to match or be a close match to snapshot but was 0.005379276499073438% different from snapshot
const config = imageSnapshotConfigurator.getConfig(fileName, 0.00006);
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: fileName,
customSnapshotsDir: getCustomSnapshotDir(fitType),
customDiffDir: join(fitDiffDir, `type-${fitType}`),
const config = imageSnapshotConfigurator.getConfig({
fileName,
fitType: afterLoadFitType,
buildCustomDiffDir: (config, fitType) => FitImageSnapshotConfigurator.buildAfterLoadDiffDir(config, fitType, onLoadFitType),
});
expect(image).toMatchImageSnapshot(config);
});

if (
(loadFitType === FitType.Center && fileName === 'with.outside.flows') ||
(loadFitType === FitType.Horizontal && fileName === 'horizontal') ||
(loadFitType === FitType.Vertical && fileName === 'vertical')
(onLoadFitType === FitType.Center && fileName === 'with.outside.flows') ||
(onLoadFitType === FitType.Horizontal && fileName === 'horizontal') ||
(onLoadFitType === FitType.Vertical && fileName === 'vertical')
) {
it.each([-100, 0, 20, 50, null])('load with margin %s', async (margin: number) => {
await initializePage({ fit: { type: loadFitType, margin: margin } }, fileName);
await initializePage({ fit: { type: onLoadFitType, margin: margin } }, fileName);

const image = await page.screenshot({ fullPage: true });

const config = imageSnapshotConfigurator.getConfig(fileName);
expect(image).toMatchImageSnapshot({
...config,
customSnapshotIdentifier: fileName,
customSnapshotsDir: getCustomSnapshotDir(loadFitType, margin),
customDiffDir: join(loadFitDiffDir, `margin-${margin}`),
const config = imageSnapshotConfigurator.getConfig({
fileName,
fitType: onLoadFitType,
margin,
buildCustomDiffDir: (config, fitType, margin) => FitImageSnapshotConfigurator.buildOnLoadDiffDir(config, fitType, true, margin),
});
expect(image).toMatchImageSnapshot(config);
});
}
});
Expand Down
20 changes: 17 additions & 3 deletions test/e2e/helpers/visu-utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ const defaultImageSnapshotConfig: MatchImageSnapshotOptions = {
};

export class ImageSnapshotConfigurator {
protected readonly defaultCustomDiffDir: string;
protected readonly defaultCustomSnapshotsDir: string;
/**
* <b>About `thresholdConfig`</b>
*
Expand All @@ -60,12 +62,17 @@ export class ImageSnapshotConfigurator {
* This is generally only required for diagram containing labels. If you are not testing the labels (value, position, ...) as part of the use case you want to cover, remove labels
* from the BPMN diagram to avoid such discrepancies.
*/
constructor(readonly thresholdConfig: Map<string, ImageSnapshotThresholdConfig>) {}
constructor(readonly thresholdConfig: Map<string, ImageSnapshotThresholdConfig>, private customDirName: string, readonly defaultFailureThreshold = 0.000004) {
this.defaultCustomDiffDir = join(ImageSnapshotConfigurator.getDiffDir(), customDirName);
this.defaultCustomSnapshotsDir = join(ImageSnapshotConfigurator.getSnapshotsDir(), customDirName);
}

// minimal threshold to make tests for diagram renders pass on local
// macOS: Expected image to match or be a close match to snapshot but was 0.00031509446166699817% different from snapshot
getConfig(fileName: string, failureThreshold = 0.000004): MatchImageSnapshotOptions {
getConfig(param: string | { fileName: string }): MatchImageSnapshotOptions {
const fileName = typeof param === 'string' ? param : param.fileName;
const config = this.thresholdConfig.get(fileName);
let failureThreshold = this.defaultFailureThreshold;
if (config) {
log(`Building dedicated image snapshot configuration for '${fileName}'`);
const simplePlatformName = getSimplePlatformName();
Expand All @@ -77,7 +84,14 @@ export class ImageSnapshotConfigurator {
}

log(`ImageSnapshot - using failureThreshold: ${failureThreshold}`);
return { ...defaultImageSnapshotConfig, failureThreshold: failureThreshold, failureThresholdType: 'percent' };
return {
...defaultImageSnapshotConfig,
failureThreshold: failureThreshold,
failureThresholdType: 'percent',
customSnapshotIdentifier: fileName,
customSnapshotsDir: this.defaultCustomSnapshotsDir,
customDiffDir: this.defaultCustomDiffDir,
};
}

static getSnapshotsDir(): string {
Expand Down