fix(core): support utf8 surrogates (#2267)

packages/core/package.json

@@ -49,7 +49,6 @@
     "ajv": "^8.6.0",
     "ajv-errors": "~3.0.0",
     "ajv-formats": "~2.1.0",
-    "blueimp-md5": "2.18.0",
     "es-aggregate-error": "^1.0.7",
     "jsonpath-plus": "7.1.0",
     "lodash": "~4.17.21",

packages/core/src/__tests__/linter.test.ts

@@ -1326,6 +1326,52 @@ responses:: !!foo
     ]);
   });
 
+  test('should handle utf8 surrogate pairs', async () => {
+    const documentUri = normalize(path.join(__dirname, './__fixtures__/test.json'));
+    const ruleset: RulesetDefinition = {
+      rules: {
+        'valid-type': {
+          given: '$..type',
+          then: {
+            function: truthy,
+          },
+        },
+      },
+    };
+
+    const spectral = new Spectral();
+    spectral.setRuleset(new Ruleset(ruleset, { source: path.join(path.dirname(documentUri), 'ruleset.json') }));
+
+    const document = new Document(
+      JSON.stringify({
+        '\uD87E\uDC04-WORKS': {
+          type: null,
+        },
+        '\uD83D-WORKS-AS-WELL': {
+          type: {
+            foo: {
+              type: null,
+            },
+          },
+        },
+      }),
+      Parsers.Json,
+      documentUri,
+    );
+
+    const results = await spectral.run(document);
+    expect(results).toEqual([
+      expect.objectContaining({
+        code: 'valid-type',
+        path: ['\uD87E\uDC04-WORKS', 'type'],
+      }),
+      expect.objectContaining({
+        code: 'valid-type',
+        path: ['\uD83D-WORKS-AS-WELL', 'type', 'foo', 'type'],
+      }),
+    ]);
+  });
+
   describe('Pointers in overrides', () => {
     test('should be supported', async () => {
       const documentUri = normalize(path.join(__dirname, './__fixtures__/test.json'));

packages/core/src/runner/runner.ts

@@ -1,7 +1,7 @@
 import { IDocument } from '../document';
 import { DocumentInventory } from '../documentInventory';
 import { IRuleResult } from '../types';
-import { ComputeFingerprintFunc, prepareResults } from '../utils';
+import { prepareResults } from './utils/results';
 import { lintNode } from './lintNode';
 import { IRunnerInternalContext } from './types';
 import { Ruleset } from '../ruleset/ruleset';
@@ -69,8 +69,8 @@ export class Runner {
     }
   }
 
-  public getResults(computeFingerprint: ComputeFingerprintFunc): IRuleResult[] {
-    return prepareResults(this.results, computeFingerprint);
+  public getResults(): IRuleResult[] {
+    return prepareResults(this.results);
   }
 }
 

packages/core/src/runner/utils/__tests__/__fixtures__/random-results.ts

@@ -0,0 +1,126 @@
+import { DiagnosticSeverity } from '@stoplight/types';
+import type { ISpectralDiagnostic } from '../../../../types';
+
+export const results: ISpectralDiagnostic[] = [
+  {
+    code: 'code 01',
+    path: ['a', 'b', 'c', 'd'],
+    source: 'source 01',
+    range: {
+      start: { line: 1, character: 1 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 01',
+    path: ['a', 'b', 'c', 'd'],
+    source: 'source 02',
+    range: {
+      start: { line: 1, character: 1 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 01',
+    path: ['a', 'b', 'c', 'd'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 1 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 01',
+    path: ['a', 'b', 'c', 'd'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 2 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 02',
+    path: ['a', 'b', 'c', 'd'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 2 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 02',
+    path: ['a', 'b', 'c', 'e'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 2 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 02',
+    path: ['a', 'b', 'c', 'f'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 2 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 03',
+    path: ['a', 'b', 'c', 'f'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 2 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 03',
+    path: ['a', 'b', 'c', 'f'],
+    source: 'source 02',
+    range: {
+      start: { line: 2, character: 3 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 03',
+    path: ['a', 'b', 'c', 'f'],
+    source: 'source 02',
+    range: {
+      start: { line: 3, character: 3 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+  {
+    code: 'code 03',
+    path: ['a', 'b', 'c', 'f'],
+    source: 'source 03',
+    range: {
+      start: { line: 3, character: 3 },
+      end: { line: 99, character: 99 },
+    },
+    message: '99',
+    severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+  },
+];

packages/core/src/runner/utils/__tests__/results.test.ts

@@ -0,0 +1,162 @@
+import { prepareResults } from '../results';
+import { DiagnosticSeverity, IPosition } from '@stoplight/types';
+
+import { comparePosition, compareResults, sortResults } from '../results';
+import type { ISpectralDiagnostic } from '../../../types';
+
+import duplicateValidationResults from './__fixtures__/duplicate-validation-results.json';
+import { results } from './__fixtures__/random-results';
+
+describe('prepareResults util', () => {
+  it('deduplicate exact validation results', () => {
+    expect(prepareResults(duplicateValidationResults)).toEqual([
+      expect.objectContaining({
+        code: 'valid-example-in-schemas',
+      }),
+      expect.objectContaining({
+        code: 'valid-schema-example-in-content',
+      }),
+    ]);
+  });
+
+  it('deduplicate exact validation results with unknown source', () => {
+    const duplicateValidationResultsWithNoSource = duplicateValidationResults.map(result => ({
+      ...result,
+      source: void 0,
+    }));
+
+    expect(prepareResults(duplicateValidationResultsWithNoSource)).toEqual([
+      expect.objectContaining({
+        code: 'valid-example-in-schemas',
+      }),
+      expect.objectContaining({
+        code: 'valid-schema-example-in-content',
+      }),
+    ]);
+  });
+
+  it('deduplicate list of only duplicates', () => {
+    const onlyDuplicates = [
+      { ...duplicateValidationResults[0] },
+      { ...duplicateValidationResults[0] },
+      { ...duplicateValidationResults[0] },
+      { ...duplicateValidationResults[0] },
+    ];
+
+    expect(prepareResults(onlyDuplicates).length).toBe(1);
+  });
+
+  it('deduplicate only exact validation results', () => {
+    // verifies that results with the same code/path but different messages will not be de-duplicated
+    expect(prepareResults(duplicateValidationResults)).toEqual([
+      expect.objectContaining({
+        code: 'valid-example-in-schemas',
+      }),
+      expect.objectContaining({
+        code: 'valid-schema-example-in-content',
+      }),
+    ]);
+  });
+});
+
+describe('sortResults', () => {
+  const shuffleBy = (arr: ISpectralDiagnostic[], indices: number[]): ISpectralDiagnostic[] => {
+    expect(indices).toHaveLength(arr.length);
+
+    const shuffled = results
+      .map<ISpectralDiagnostic & { pos?: number }>((v, i) => ({ ...v, pos: indices[i] }))
+      .sort((a, b) => a.pos! - b.pos!)
+      .map(v => {
+        delete v.pos;
+        return v;
+      });
+
+    return shuffled;
+  };
+
+  test('should properly order results', () => {
+    const randomlySortedIndices = [5, 4, 1, 10, 8, 6, 3, 9, 2, 0, 7];
+
+    const shuffled = shuffleBy(results, randomlySortedIndices);
+
+    expect(sortResults(shuffled)).toEqual(results);
+  });
+});
+
+describe('compareResults', () => {
+  test('should properly order results source', () => {
+    const input = {
+      code: 'code 01',
+      path: ['a', 'b', 'c', 'd'],
+      range: {
+        start: { line: 1, character: 1 },
+        end: { line: 99, character: 99 },
+      },
+      message: '99',
+      severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+    };
+
+    [
+      { one: void 0, another: void 0, expected: 0 },
+      { one: 'a', another: void 0, expected: 1 },
+      { one: void 0, another: 'a', expected: -1 },
+      { one: 'a', another: 'a', expected: 0 },
+      { one: 'a', another: 'b', expected: -1 },
+    ].forEach(tc => {
+      expect(compareResults({ ...input, source: tc.one }, { ...input, source: tc.another })).toEqual(tc.expected);
+    });
+  });
+
+  test('should properly order results code', () => {
+    const input = {
+      source: 'somewhere',
+      path: ['a', 'b', 'c', 'd'],
+      range: {
+        start: { line: 1, character: 1 },
+        end: { line: 99, character: 99 },
+      },
+      message: '99',
+      severity: DiagnosticSeverity.Error, // or any other level, it's irrelevant
+    };
+
+    [
+      { one: 'a', another: 'a', expected: 0 },
+      { one: 'a', another: 'b', expected: -1 },
+      { one: '2', another: '10', expected: -1 },
+      { one: 1, another: 1, expected: 0 },
+      { one: 1, another: 2, expected: -1 },
+      { one: 1, another: '1', expected: 0 },
+      { one: 2, another: '10', expected: -1 },
+    ].forEach(tc => {
+      expect(compareResults({ ...input, code: tc.one }, { ...input, code: tc.another })).toEqual(tc.expected);
+    });
+  });
+});
+
+const buildPosition = (line: number, char: number): IPosition => {
+  return { line, character: char };
+};
+
+describe('comparePosition', () => {
+  const positionTestCases = [
+    [2, 2, 1, 1, 1],
+    [2, 1, 1, 1, 1],
+    [1, 2, 1, 1, 1],
+
+    [1, 1, 1, 1, 0],
+    [1, 2, 1, 2, 0],
+    [2, 1, 2, 1, 0],
+    [2, 2, 2, 2, 0],
+
+    [1, 1, 1, 2, -1],
+    [1, 1, 2, 1, -1],
+    [1, 1, 2, 2, -1],
+  ];
+
+  test.each(positionTestCases)(
+    'should properly order locations (%i, %i) vs (%i, %i)',
+    (leftLine, leftChar, rightLine, rightChar, expected) => {
+      expect(comparePosition(buildPosition(leftLine, leftChar), buildPosition(rightLine, rightChar))).toEqual(expected);
+    },
+  );
+});