From 4cd1eeee6774fd183dbc60d410754efc1aa89546 Mon Sep 17 00:00:00 2001 From: PagoNxt-Trade Date: Wed, 23 Nov 2022 12:04:04 +0100 Subject: [PATCH] feat: scoring data tests and readme documentation added --- docs/guides/2-cli.md | 43 ++++++ .../cli/src/commands/__tests__/lint.test.ts | 20 ++- packages/cli/src/commands/lint.ts | 5 +- packages/cli/src/formatters/json.ts | 6 +- packages/cli/src/formatters/pretty.ts | 6 + packages/cli/src/formatters/stylish.ts | 6 + packages/cli/src/formatters/types.ts | 4 +- .../__fixtures__/scoring-matrix.json | 18 +++ .../cli/src/services/__tests__/linter.test.ts | 17 +++ .../cli/src/services/__tests__/output.test.ts | 46 ++++++ ...sults-default-format-scoring-json.scenario | 134 ++++++++++++++++++ .../formats/results-default-scoring.scenario | 81 +++++++++++ .../results-format-stylish-scoring.scenario | 82 +++++++++++ .../formats/too-few-outputs.scenario | 1 + .../formats/too-many-outputs.scenario | 1 + .../formats/unmatched-outputs.scenario | 1 + .../scenarios/help-no-document.scenario | 1 + .../overrides/aliases-scoring.scenario | 119 ++++++++++++++++ .../fail-on-error-no-error-scoring.scenario | 50 +++++++ .../severity/fail-on-error-scoring.scenario | 64 +++++++++ .../scenarios/strict-options.scenario | 1 + .../valid-no-errors.oas2-scoring.scenario | 42 ++++++ .../scenarios/valid-no-errors.oas2.scenario | 2 +- 23 files changed, 741 insertions(+), 9 deletions(-) create mode 100644 packages/cli/src/services/__tests__/__fixtures__/scoring-matrix.json create mode 100644 test-harness/scenarios/formats/results-default-format-scoring-json.scenario create mode 100644 test-harness/scenarios/formats/results-default-scoring.scenario create mode 100644 test-harness/scenarios/formats/results-format-stylish-scoring.scenario create mode 100644 test-harness/scenarios/overrides/aliases-scoring.scenario create mode 100644 test-harness/scenarios/severity/fail-on-error-no-error-scoring.scenario create mode 100644 test-harness/scenarios/severity/fail-on-error-scoring.scenario create mode 100644 test-harness/scenarios/valid-no-errors.oas2-scoring.scenario diff --git a/docs/guides/2-cli.md b/docs/guides/2-cli.md index 846b7aa933..1fbaa7e0f6 100644 --- a/docs/guides/2-cli.md +++ b/docs/guides/2-cli.md @@ -39,6 +39,7 @@ Other options include: --stdin-filepath path to a file to pretend that stdin comes from [string] --resolver path to custom json-ref-resolver instance [string] -r, --ruleset path/URL to a ruleset file [string] + -s, --scoring-matrix path/URL to a scoring matrix config file [string] -F, --fail-severity results of this level or above will trigger a failure exit code [string] [choices: "error", "warn", "info", "hint"] [default: "error"] -D, --display-only-failures only output results equal to or greater than --fail-severity [boolean] [default: false] @@ -60,6 +61,48 @@ Here you can build a [custom ruleset](../getting-started/3-rulesets.md), or exte - [OpenAPI ruleset](../reference/openapi-rules.md) - [AsyncAPI ruleset](../reference/asyncapi-rules.md) +## Using a Scoring Matrix File + +If you want to specify a custom scoring matrix and some results count customization, you can add a config file in json format in that way: + +```bash +spectral lint ./reference/**/*.oas*.{json,yml,yaml} --scoring-matrix ./scoringFile.json +``` +or +```bash +spectral lint ./reference/**/*.oas*.{json,yml,yaml} -s ./scoringFile.json +``` + +Heres an example of this config file: + +``` + { + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A":75, + "B":65, + "C":55, + "D":45, + "E":0 + }, + "threshold":50, + "warningsSubtract": true, + "uniqueErrors": false + } +``` + +Where: + - scoringSubctract : An object with an array for every result level we want to substract percentage, with the perctentage to subsctrac from 0 to 10 on every result type + - scoringLetter : An object with key/value pairs with scoring letter and scoring percentage, that the result must be greater , for this letter + - threshold : A number with minimum percentage value to provide valid the file we are checking + - warningsSubtract : A boolean to setup if accumulate the result types to less the scorgin percentage or stop counting on most critical result types + - uniqueErrors : A boolean to setup a count with unique errors or with all of them + ## Error Results Spectral has a few different error severities: `error`, `warn`, `info`, and `hint`, and they are in "order" from highest to lowest. By default, all results will be shown regardless of severity, but since v5.0, only the presence of errors will cause a failure status code of 1. Seeing results and getting a failure code for it are now two different things. diff --git a/packages/cli/src/commands/__tests__/lint.test.ts b/packages/cli/src/commands/__tests__/lint.test.ts index eb98e7050d..1ccd70db7f 100644 --- a/packages/cli/src/commands/__tests__/lint.test.ts +++ b/packages/cli/src/commands/__tests__/lint.test.ts @@ -146,6 +146,22 @@ describe('lint', () => { ); }); + it('calls lint with document, ruleset and scoring matrix config file', async () => { + const doc = './__fixtures__/empty-oas2-document.json'; + const ruleset = 'custom-ruleset.json'; + const configFile = 'scoring-matrix.json'; + await run(`lint -r ${ruleset} -s ${configFile} ${doc}`); + expect(lint).toBeCalledWith([doc], { + encoding: 'utf8', + format: ['stylish'], + output: { stylish: '' }, + ruleset: 'custom-ruleset.json', + stdinFilepath: undefined, + ignoreUnknownFormat: false, + failOnUnmatchedGlobs: false, + }); + }); + it.each(['json', 'stylish'])('calls formatOutput with %s format', async format => { await run(`lint -f ${format} ./__fixtures__/empty-oas2-document.json`); expect(formatOutput).toBeCalledWith(results, format, { failSeverity: DiagnosticSeverity.Error }); @@ -244,13 +260,13 @@ describe('lint', () => { expect(process.stderr.write).nthCalledWith(2, `Error #1: ${chalk.red('some unhandled exception')}\n`); expect(process.stderr.write).nthCalledWith( 3, - expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:236`), + expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:252`), ); expect(process.stderr.write).nthCalledWith(4, `Error #2: ${chalk.red('another one')}\n`); expect(process.stderr.write).nthCalledWith( 5, - expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:237`), + expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:253`), ); expect(process.stderr.write).nthCalledWith(6, `Error #3: ${chalk.red('original exception')}\n`); diff --git a/packages/cli/src/commands/lint.ts b/packages/cli/src/commands/lint.ts index eacb383112..4e79efbad6 100644 --- a/packages/cli/src/commands/lint.ts +++ b/packages/cli/src/commands/lint.ts @@ -14,7 +14,7 @@ import { formatOutput, writeOutput } from '../services/output'; import { FailSeverity, ILintConfig, OutputFormat } from '../services/config'; import { CLIError } from '../errors'; -import { getScoringMatrix } from '../formatters/utils'; +import { getScoringMatrix } from '../formatters/utils/getScoring'; const formatOptions = Object.values(OutputFormat); @@ -128,11 +128,10 @@ const lintCommand: CommandModule = { description: 'path/URL to a ruleset file', type: 'string', }, - scoringMatrix: { + 'scoring-matrix': { alias: 's', description: 'path/URL to a scoring matrix config file', type: 'string', - }, 'fail-severity': { alias: 'F', diff --git a/packages/cli/src/formatters/json.ts b/packages/cli/src/formatters/json.ts index d9addfb7b5..c8068aa5eb 100644 --- a/packages/cli/src/formatters/json.ts +++ b/packages/cli/src/formatters/json.ts @@ -10,7 +10,9 @@ export const json: Formatter = (results: ISpectralDiagnostic[], options: Formatt let groupedResults; let scoringText = ''; if (options.scoringMatrix !== void 0) { - spectralVersion = options.scoringMatrix.customScoring + (version as string); + if (options.scoringMatrix.customScoring !== undefined) { + spectralVersion = `${options.scoringMatrix.customScoring} ${version as string}`; + } groupedResults = groupBySource(uniqueErrors(results)); scoringText = getScoringText(getCountsBySeverity(groupedResults), options.scoringMatrix); } @@ -26,9 +28,11 @@ export const json: Formatter = (results: ISpectralDiagnostic[], options: Formatt }); let objectOutput; if (options.scoringMatrix !== void 0) { + const scoring = +(scoringText !== null ? scoringText.replace('%', '').split(/[()]+/)[1] : 0); objectOutput = { version: spectralVersion, scoring: scoringText.replace('SCORING:', '').trim(), + passed: scoring >= options.scoringMatrix.threshold, results: outputJson, }; } else { diff --git a/packages/cli/src/formatters/pretty.ts b/packages/cli/src/formatters/pretty.ts index 50dc836a4c..962659f434 100644 --- a/packages/cli/src/formatters/pretty.ts +++ b/packages/cli/src/formatters/pretty.ts @@ -116,6 +116,12 @@ export const pretty: Formatter = (results: ISpectralDiagnostic[], options: Forma output += chalk[summaryColor].bold(`\u2716${summaryText !== null ? ` ${summaryText}` : ''}\n`); if (options.scoringMatrix !== void 0) { output += chalk[scoringColor].bold(`\u2716${scoringText !== null ? ` ${scoringText}` : ''}\n`); + const scoring = +(scoringText !== null ? scoringText.replace('%', '').split(/[()]+/)[1] : 0); + if (scoring >= options.scoringMatrix.threshold) { + output += chalk['green'].bold(`\u2716 PASSED!\n`); + } else { + output += chalk['red'].bold(`\u2716 NOT PASSED!\n`); + } } return output; diff --git a/packages/cli/src/formatters/stylish.ts b/packages/cli/src/formatters/stylish.ts index 2f2e418cd2..ed1a9eea80 100644 --- a/packages/cli/src/formatters/stylish.ts +++ b/packages/cli/src/formatters/stylish.ts @@ -126,6 +126,12 @@ export const stylish: Formatter = (results: ISpectralDiagnostic[], options: Form output += chalk[summaryColor].bold(`\u2716 ${summaryText}\n`); if (options.scoringMatrix !== void 0) { output += chalk[scoringColor].bold(`\u2716${scoringText !== null ? ` ${scoringText}` : ''}\n`); + const scoring = +(scoringText !== null ? scoringText.replace('%', '').split(/[()]+/)[1] : 0); + if (scoring >= options.scoringMatrix.threshold) { + output += chalk['green'].bold(`\u2716 PASSED!\n`); + } else { + output += chalk['red'].bold(`\u2716 NOT PASSED!\n`); + } } return output; diff --git a/packages/cli/src/formatters/types.ts b/packages/cli/src/formatters/types.ts index a4904fe187..58ef1fea44 100644 --- a/packages/cli/src/formatters/types.ts +++ b/packages/cli/src/formatters/types.ts @@ -5,11 +5,11 @@ import type { DiagnosticSeverity } from '@stoplight/types'; export type ScoringTable = { [key in HumanReadableDiagnosticSeverity]: number[]; }; -interface ScoringLevel { +export interface ScoringLevel { [key: string]: number; } export type ScoringMatrix = { - customScoring: string; + customScoring?: string; scoringSubtract: ScoringTable[]; scoringLetter: ScoringLevel[]; threshold: number; diff --git a/packages/cli/src/services/__tests__/__fixtures__/scoring-matrix.json b/packages/cli/src/services/__tests__/__fixtures__/scoring-matrix.json new file mode 100644 index 0000000000..c1bc9dbcc0 --- /dev/null +++ b/packages/cli/src/services/__tests__/__fixtures__/scoring-matrix.json @@ -0,0 +1,18 @@ +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} \ No newline at end of file diff --git a/packages/cli/src/services/__tests__/linter.test.ts b/packages/cli/src/services/__tests__/linter.test.ts index e17a1b77e3..c48d4de26e 100644 --- a/packages/cli/src/services/__tests__/linter.test.ts +++ b/packages/cli/src/services/__tests__/linter.test.ts @@ -18,6 +18,7 @@ jest.mock('../output'); const validCustomOas3SpecPath = resolve(__dirname, '__fixtures__/openapi-3.0-valid-custom.yaml'); const invalidRulesetPath = resolve(__dirname, '__fixtures__/ruleset-invalid.js'); const validRulesetPath = resolve(__dirname, '__fixtures__/ruleset-valid.js'); +const validScoringMatrixRulesetPath = resolve(__dirname, '__fixtures__/scorint-matrix.json'); const validOas3SpecPath = resolve(__dirname, './__fixtures__/openapi-3.0-valid.yaml'); async function run(command: string) { @@ -368,6 +369,22 @@ describe('Linter service', () => { }); }); + describe('--scoring-matrix ', () => { + describe('when single scoring-matrix option provided', () => { + it('outputs normal output if it does not exist', () => { + return expect( + run(`lint ${validCustomOas3SpecPath} -r ${validRulesetPath} -s non-existent-path`), + ).resolves.toEqual([]); + }); + + it('outputs no issues', () => { + return expect( + run(`lint ${validCustomOas3SpecPath} -r ${validRulesetPath} -s ${validScoringMatrixRulesetPath}`), + ).resolves.toEqual([]); + }); + }); + }); + describe('when loading specification files from web', () => { it('outputs no issues', () => { const document = join(__dirname, `./__fixtures__/stoplight-info-document.json`); diff --git a/packages/cli/src/services/__tests__/output.test.ts b/packages/cli/src/services/__tests__/output.test.ts index c9f08e305e..20fe5fd268 100644 --- a/packages/cli/src/services/__tests__/output.test.ts +++ b/packages/cli/src/services/__tests__/output.test.ts @@ -2,6 +2,7 @@ import { DiagnosticSeverity } from '@stoplight/types'; import * as fs from 'fs'; import * as process from 'process'; import * as formatters from '../../formatters'; +import { ScoringLevel, ScoringTable } from '../../formatters/types'; import { OutputFormat } from '../config'; import { formatOutput, writeOutput } from '../output'; @@ -14,6 +15,23 @@ jest.mock('fs', () => ({ })); jest.mock('process'); +const scoringMatrix = { + scoringSubtract: { + error: [0, 55, 65, 75, 75, 75, 85, 85, 85, 85, 95], + warn: [0, 3, 7, 10, 10, 10, 15, 15, 15, 15, 18], + } as unknown as ScoringTable[], + scoringLetter: { + A: 75, + B: 65, + C: 55, + D: 45, + E: 0, + } as unknown as ScoringLevel[], + threshold: 50, + warningsSubtract: true, + uniqueErrors: false, +}; + describe('Output service', () => { describe('formatOutput', () => { it.each(['stylish', 'json', 'junit'])('calls %s formatter with given result', format => { @@ -41,6 +59,34 @@ describe('Output service', () => { (formatters[format] as jest.Mock).mockReturnValueOnce(output); expect(formatOutput(results, format as OutputFormat, { failSeverity: DiagnosticSeverity.Error })).toEqual(output); }); + + it.each(['stylish', 'json', 'pretty'])('calls %s formatter with given result and scoring-matrix', format => { + const results = [ + { + code: 'info-contact', + path: ['info'], + message: 'Info object should contain `contact` object.', + severity: DiagnosticSeverity.Information, + range: { + start: { + line: 2, + character: 9, + }, + end: { + line: 6, + character: 19, + }, + }, + source: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json', + }, + ]; + + const output = `value for ${format}`; + (formatters[format] as jest.Mock).mockReturnValueOnce(output); + expect( + formatOutput(results, format as OutputFormat, { failSeverity: DiagnosticSeverity.Error, scoringMatrix }), + ).toEqual(output); + }); }); describe('writeOutput', () => { diff --git a/test-harness/scenarios/formats/results-default-format-scoring-json.scenario b/test-harness/scenarios/formats/results-default-format-scoring-json.scenario new file mode 100644 index 0000000000..7dc42570a5 --- /dev/null +++ b/test-harness/scenarios/formats/results-default-format-scoring-json.scenario @@ -0,0 +1,134 @@ +====test==== +Invalid document outputs results with scoring data --format=json +====document==== +--- +info: + version: 1.0.0 + title: Stoplight +====asset:ruleset.json==== + { + "rules": { + "api-servers": { + "description": "\"servers\" must be present and non-empty array.", + "recommended": true, + "given": "$", + "then": { + "field": "servers", + "function": "schema", + "functionOptions": { + "dialect": "draft7", + "schema": { + "items": { + "type": "object", + }, + "minItems": 1, + "type": "array" + } + } + } + }, + "info-contact": { + "description": "Info object must have a \"contact\" object.", + "recommended": true, + "type": "style", + "given": "$", + "then": { + "field": "info.contact", + "function": "truthy", + } + }, + "info-description": { + "description": "Info \"description\" must be present and non-empty string.", + "recommended": true, + "type": "style", + "given": "$", + "then": { + "field": "info.description", + "function": "truthy" + } + } + } + } +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====command==== +{bin} lint {document} --format=json --ruleset "{asset:ruleset.json}" --scoring-matrix "{asset:scoring-matrix.json}" +====stdout==== +{ + "version": "", + "scoring": "A (90%)", + "passed": true, + "results": [ + { + "code": "api-servers", + "path": [], + "message": "\"servers\" must be present and non-empty array.", + "severity": 1, + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 3, + "character": 18 + } + }, + "source": "{document}" + }, + { + "code": "info-contact", + "path": [ + "info" + ], + "message": "Info object must have a \"contact\" object.", + "severity": 1, + "range": { + "start": { + "line": 1, + "character": 5 + }, + "end": { + "line": 3, + "character": 18 + } + }, + "source": "{document}" + }, + { + "code": "info-description", + "path": [ + "info" + ], + "message": "Info \"description\" must be present and non-empty string.", + "severity": 1, + "range": { + "start": { + "line": 1, + "character": 5 + }, + "end": { + "line": 3, + "character": 18 + } + }, + "source": "{document}" + } + ]} diff --git a/test-harness/scenarios/formats/results-default-scoring.scenario b/test-harness/scenarios/formats/results-default-scoring.scenario new file mode 100644 index 0000000000..0e9a1af829 --- /dev/null +++ b/test-harness/scenarios/formats/results-default-scoring.scenario @@ -0,0 +1,81 @@ +====test==== +Invalid document returns results with scoring data in default (stylish) format +====document==== +--- +info: + version: 1.0.0 + title: Stoplight +====asset:ruleset.json==== +{ + "rules": { + "api-servers": { + "description": "\"servers\" must be present and non-empty array.", + "recommended": true, + "given": "$", + "then": { + "field": "servers", + "function": "schema", + "functionOptions": { + "dialect": "draft7", + "schema": { + "items": { + "type": "object", + }, + "minItems": 1, + "type": "array" + } + } + } + }, + "info-contact": { + "description": "Info object must have a \"contact\" object.", + "recommended": true, + "type": "style", + "given": "$", + "then": { + "field": "info.contact", + "function": "truthy", + } + }, + "info-description": { + "description": "Info \"description\" must be present and non-empty string.", + "recommended": true, + "type": "style", + "given": "$", + "then": { + "field": "info.description", + "function": "truthy" + } + } + } +} +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====command==== +{bin} lint {document} --ruleset "{asset:ruleset.json}" --scoring-matrix "{asset:scoring-matrix.json}" +====stdout==== +{document} + 1:1 warning api-servers "servers" must be present and non-empty array. + 2:6 warning info-contact Info object must have a "contact" object. info + 2:6 warning info-description Info "description" must be present and non-empty string. info + +✖ 3 problems (0 errors, 3 warnings, 0 infos, 0 hints) +✖ SCORING: A (90%) +✖ PASSED! diff --git a/test-harness/scenarios/formats/results-format-stylish-scoring.scenario b/test-harness/scenarios/formats/results-format-stylish-scoring.scenario new file mode 100644 index 0000000000..fbc8318c6d --- /dev/null +++ b/test-harness/scenarios/formats/results-format-stylish-scoring.scenario @@ -0,0 +1,82 @@ +====test==== +Invalid document outputs results with scoring data when --format=stylish +====document==== +--- +info: + version: 1.0.0 + title: Stoplight +paths: {} +====asset:ruleset.json==== +{ + "rules": { + "api-servers": { + "description": "\"servers\" must be present and non-empty array.", + "recommended": true, + "given": "$", + "then": { + "field": "servers", + "function": "schema", + "functionOptions": { + "dialect": "draft7", + "schema": { + "items": { + "type": "object", + }, + "minItems": 1, + "type": "array" + } + } + } + }, + "info-contact": { + "description": "Info object must have a \"contact\" object.", + "recommended": true, + "type": "style", + "given": "$", + "then": { + "field": "info.contact", + "function": "truthy", + } + }, + "info-description": { + "description": "Info \"description\" must be present and non-empty string.", + "recommended": true, + "type": "style", + "given": "$", + "then": { + "field": "info.description", + "function": "truthy" + } + } + } +} +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====command==== +{bin} lint {document} --format=stylish --ruleset "{asset:ruleset.json}" --scoring-matrix "{asset:scoring-matrix.json}" +====stdout==== +{document} + 1:1 warning api-servers "servers" must be present and non-empty array. + 2:6 warning info-contact Info object must have a "contact" object. info + 2:6 warning info-description Info "description" must be present and non-empty string. info + +✖ 3 problems (0 errors, 3 warnings, 0 infos, 0 hints) +✖ SCORING: A (90%) +✖ PASSED! diff --git a/test-harness/scenarios/formats/too-few-outputs.scenario b/test-harness/scenarios/formats/too-few-outputs.scenario index 733e541854..6695182f91 100644 --- a/test-harness/scenarios/formats/too-few-outputs.scenario +++ b/test-harness/scenarios/formats/too-few-outputs.scenario @@ -24,6 +24,7 @@ Options: --stdin-filepath path to a file to pretend that stdin comes from [string] --resolver path to custom json-ref-resolver instance [string] -r, --ruleset path/URL to a ruleset file [string] + -s, --scoring-matrix path/URL to a scoring matrix config file [string] -F, --fail-severity results of this level or above will trigger a failure exit code [string] [choices: "error", "warn", "info", "hint"] [default: "error"] -D, --display-only-failures only output results equal to or greater than --fail-severity [boolean] [default: false] --ignore-unknown-format do not warn about unmatched formats [boolean] [default: false] diff --git a/test-harness/scenarios/formats/too-many-outputs.scenario b/test-harness/scenarios/formats/too-many-outputs.scenario index c127e994af..6484ea7d61 100644 --- a/test-harness/scenarios/formats/too-many-outputs.scenario +++ b/test-harness/scenarios/formats/too-many-outputs.scenario @@ -24,6 +24,7 @@ Options: --stdin-filepath path to a file to pretend that stdin comes from [string] --resolver path to custom json-ref-resolver instance [string] -r, --ruleset path/URL to a ruleset file [string] + -s, --scoring-matrix path/URL to a scoring matrix config file [string] -F, --fail-severity results of this level or above will trigger a failure exit code [string] [choices: "error", "warn", "info", "hint"] [default: "error"] -D, --display-only-failures only output results equal to or greater than --fail-severity [boolean] [default: false] --ignore-unknown-format do not warn about unmatched formats [boolean] [default: false] diff --git a/test-harness/scenarios/formats/unmatched-outputs.scenario b/test-harness/scenarios/formats/unmatched-outputs.scenario index 69f7f1fc5e..4dc9714ca7 100644 --- a/test-harness/scenarios/formats/unmatched-outputs.scenario +++ b/test-harness/scenarios/formats/unmatched-outputs.scenario @@ -24,6 +24,7 @@ Options: --stdin-filepath path to a file to pretend that stdin comes from [string] --resolver path to custom json-ref-resolver instance [string] -r, --ruleset path/URL to a ruleset file [string] + -s, --scoring-matrix path/URL to a scoring matrix config file [string] -F, --fail-severity results of this level or above will trigger a failure exit code [string] [choices: "error", "warn", "info", "hint"] [default: "error"] -D, --display-only-failures only output results equal to or greater than --fail-severity [boolean] [default: false] --ignore-unknown-format do not warn about unmatched formats [boolean] [default: false] diff --git a/test-harness/scenarios/help-no-document.scenario b/test-harness/scenarios/help-no-document.scenario index 8e686198b5..5502ecd0a6 100644 --- a/test-harness/scenarios/help-no-document.scenario +++ b/test-harness/scenarios/help-no-document.scenario @@ -25,6 +25,7 @@ Options: --stdin-filepath path to a file to pretend that stdin comes from [string] --resolver path to custom json-ref-resolver instance [string] -r, --ruleset path/URL to a ruleset file [string] + -s, --scoring-matrix path/URL to a scoring matrix config file [string] -F, --fail-severity results of this level or above will trigger a failure exit code [string] [choices: "error", "warn", "info", "hint"] [default: "error"] -D, --display-only-failures only output results equal to or greater than --fail-severity [boolean] [default: false] --ignore-unknown-format do not warn about unmatched formats [boolean] [default: false] diff --git a/test-harness/scenarios/overrides/aliases-scoring.scenario b/test-harness/scenarios/overrides/aliases-scoring.scenario new file mode 100644 index 0000000000..9a5033b5b1 --- /dev/null +++ b/test-harness/scenarios/overrides/aliases-scoring.scenario @@ -0,0 +1,119 @@ +====test==== +Respect overrides with aliases and scoring +====asset:spectral.js==== +const { DiagnosticSeverity } = require('@stoplight/types'); +const { pattern } = require('@stoplight/spectral-functions'); + +module.exports = { + aliases: { + Info: ['$.info'], + }, + rules: { + 'description-matches-stoplight': { + message: 'Description must contain Stoplight', + given: '#Info', + recommended: true, + severity: DiagnosticSeverity.Error, + then: { + field: 'description', + function: pattern, + functionOptions: { + match: 'Stoplight', + }, + }, + }, + 'title-matches-stoplight': { + message: 'Title must contain Stoplight', + given: '#Info', + then: { + field: 'title', + function: pattern, + functionOptions: { + match: 'Stoplight', + }, + }, + }, + 'contact-name-matches-stoplight': { + message: 'Contact name must contain Stoplight', + given: '#Info.contact', + recommended: false, + then: { + field: 'name', + function: pattern, + functionOptions: { + match: 'Stoplight', + }, + }, + }, + }, + overrides: [ + { + files: [`**/*.json`], + rules: { + 'description-matches-stoplight': 'error', + 'title-matches-stoplight': 'warn', + }, + }, + { + files: [`v2/**/*.json`], + rules: { + 'description-matches-stoplight': 'info', + 'title-matches-stoplight': 'hint', + }, + }, + ], +}; +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====asset:v2/document.json==== +{ + "info": { + "description": "", + "title": "", + "contact": { + "name": "" + } + } +} +====asset:legacy/document.json==== +{ + "info": { + "description": "", + "title": "", + "contact": { + "name": "" + } + } +} +====command==== +{bin} lint **/*.json --ruleset {asset:spectral.js} --fail-on-unmatched-globs --scoring-matrix "{asset:scoring-matrix.json}" +====stdout==== + +{asset:legacy/document.json} + 3:20 error description-matches-stoplight Description must contain Stoplight info.description + 4:14 warning title-matches-stoplight Title must contain Stoplight info.title + +{asset:v2/document.json} + 3:20 information description-matches-stoplight Description must contain Stoplight info.description + 4:14 hint title-matches-stoplight Title must contain Stoplight info.title + +✖ 4 problems (1 error, 1 warning, 1 info, 1 hint) +✖ SCORING: A (88%) +✖ PASSED! diff --git a/test-harness/scenarios/severity/fail-on-error-no-error-scoring.scenario b/test-harness/scenarios/severity/fail-on-error-no-error-scoring.scenario new file mode 100644 index 0000000000..b656af75b2 --- /dev/null +++ b/test-harness/scenarios/severity/fail-on-error-no-error-scoring.scenario @@ -0,0 +1,50 @@ +====test==== +Will only fail if there is an error, and there is not. Can still see all warnings with scoring data. +====document==== +- type: string +- type: number +====asset:ruleset.json==== +{ + "rules": { + "valid-type": { + "given": "$..type", + "then": { + "function": "enumeration", + "functionOptions": { + "values": ["object"] + } + } + } + } +} +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====command==== +{bin} lint {document} --ruleset "{asset:ruleset.json}" --fail-severity=error --scoring-matrix "{asset:scoring-matrix.json}" +====status==== +0 +====stdout==== +{document} + 1:9 warning valid-type "string" must be equal to one of the allowed values: "object" [0].type + 2:9 warning valid-type "number" must be equal to one of the allowed values: "object" [1].type + +✖ 2 problems (0 errors, 2 warnings, 0 infos, 0 hints) +✖ SCORING: A (93%) +✖ PASSED! diff --git a/test-harness/scenarios/severity/fail-on-error-scoring.scenario b/test-harness/scenarios/severity/fail-on-error-scoring.scenario new file mode 100644 index 0000000000..d2f18d95b6 --- /dev/null +++ b/test-harness/scenarios/severity/fail-on-error-scoring.scenario @@ -0,0 +1,64 @@ +====test==== +Will fail and return 1 as exit code because errors exist with scoring data +====document==== +- type: string +- type: array +====asset:ruleset.json==== +{ + "rules": { + "valid-type": { + "given": "$..type", + "severity": "error", + "then": { + "function": "enumeration", + "functionOptions": { + "values": ["object"] + } + } + }, + "no-primitive-type": { + "given": "$..type", + "severity": "warn", + "then": { + "function": "enumeration", + "functionOptions": { + "values": ["string", "number", "boolean", "null"] + } + } + } + } +} +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====command-nix==== +{bin} lint {document} --ruleset "{asset:ruleset.json}" --fail-severity=error --scoring-matrix "{asset:scoring-matrix.json}" +====command-win==== +{bin} lint {document} --ruleset "{asset:ruleset.json}" --fail-severity error --scoring-matrix "{asset:scoring-matrix.json}" +====status==== +1 +====stdout==== +{document} + 1:9 error valid-type "string" must be equal to one of the allowed values: "object" [0].type + 2:9 warning no-primitive-type "array" must be equal to one of the allowed values: "string", "number", "boolean", "null" [1].type + 2:9 error valid-type "array" must be equal to one of the allowed values: "object" [1].type + +✖ 3 problems (2 errors, 1 warning, 0 infos, 0 hints) +✖ SCORING: A (90%) +✖ PASSED! diff --git a/test-harness/scenarios/strict-options.scenario b/test-harness/scenarios/strict-options.scenario index 8b1cb37084..ba2e254026 100644 --- a/test-harness/scenarios/strict-options.scenario +++ b/test-harness/scenarios/strict-options.scenario @@ -25,6 +25,7 @@ Options: --stdin-filepath path to a file to pretend that stdin comes from [string] --resolver path to custom json-ref-resolver instance [string] -r, --ruleset path/URL to a ruleset file [string] + -s, --scoring-matrix path/URL to a scoring matrix config file [string] -F, --fail-severity results of this level or above will trigger a failure exit code [string] [choices: "error", "warn", "info", "hint"] [default: "error"] -D, --display-only-failures only output results equal to or greater than --fail-severity [boolean] [default: false] --ignore-unknown-format do not warn about unmatched formats [boolean] [default: false] diff --git a/test-harness/scenarios/valid-no-errors.oas2-scoring.scenario b/test-harness/scenarios/valid-no-errors.oas2-scoring.scenario new file mode 100644 index 0000000000..27ef2db366 --- /dev/null +++ b/test-harness/scenarios/valid-no-errors.oas2-scoring.scenario @@ -0,0 +1,42 @@ +====test==== +Valid OAS2 document returns no results +====document==== +swagger: "2.0" +info: + version: 1.0.0 + title: Stoplight + description: lots of text + contact: + name: fred +host: localhost +schemes: + - http +paths: {} +tags: + - name: my-tag +====asset:ruleset==== +const { oas } = require('@stoplight/spectral-rulesets'); +module.exports = oas; +====asset:scoring-matrix.json==== +{ + "scoringSubtract": + { + "error": [ 0,55,65,75,75,75,85,85,85,85,95 ], + "warn": [ 0,3,7,10,10,10,15,15,15,15,18 ] + }, + "scoringLetter": + { + "A": 75, + "B": 65, + "C": 55, + "D": 45, + "E": 0 + }, + "threshold": 50, + "warningsSubtract": true, + "uniqueErrors": false +} +====command==== +{bin} lint {document} --ruleset "{asset:ruleset}" --scoring-matrix "{asset:scoring-matrix.json}" +====stdout==== +No results with a severity of 'error' found! diff --git a/test-harness/scenarios/valid-no-errors.oas2.scenario b/test-harness/scenarios/valid-no-errors.oas2.scenario index b671062f2e..f2db9703c3 100644 --- a/test-harness/scenarios/valid-no-errors.oas2.scenario +++ b/test-harness/scenarios/valid-no-errors.oas2.scenario @@ -9,7 +9,7 @@ info: contact: name: fred host: localhost -schemes: +schemes: - http paths: {} tags: