Initial commit of override tooling (#4158)

* Initial commit of override tooling

Add a foundation for new override tooling described in #4104. This includes:

- Build scripts, lint scripts, config files, etc
- Logic for parsing and checking validity of an override manifest
- Unit tests for override manifest logic
- Abstractions to allow fetching React Native files of arbtrary versions

A lot of this is foundational. The override logic has been well-tested,
and the Git logic has been manually tested, but we don't have much
end-to-end set up yet.

* Address comments and deuplicate lockfile

* Add more dependencies for WebDriverIO

We hardcode an old version of WebderiverIO beacuse of #3019. These seem to have loose dependency requirements, because the change to deuplicate packages broke this (see webdriverio/webdriverio#4104). Hardcode resolutions in E2ETest for existing versions of wdio packages in the meantime.

package.json

@@ -35,6 +35,10 @@
     "lerna": "^3.16.1"
   },
   "resolutions": {
-    "**/eslint-plugin-react": "^7.14.1"
+    "**/eslint-plugin-react": "^7.14.1",
+    "e2etest/@wdio/config": "5.13.2",
+    "e2etest/@wdio/protocols": "5.13.2",
+    "e2etest/@wdio/repl": "5.13.2",
+    "e2etest/@wdio/utils": "5.13.2"
   }
 }

packages/override-tools/.eslintignore

@@ -0,0 +1,3 @@
+lib/
+lib-commonjs/
+node_modules/

packages/override-tools/.eslintrc.js

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ * 
+ * @format
+ * @ts-check
+ */
+
+module.exports = { 
+  extends: "@react-native-community",
+  rules:{ 
+    "sort-imports": "warn",
+}};

packages/override-tools/.gitignore

@@ -0,0 +1,2 @@
+lib/
+lib-commonjs/

packages/override-tools/.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug Jest Tests",
+            "type": "node",
+            "request": "launch",
+            "runtimeArgs": [
+              "--inspect-brk",
+              "${workspaceRoot}/../../node_modules/jest/bin/jest.js",
+              "--runInBand"
+            ],
+            "console": "integratedTerminal",
+            "internalConsoleOptions": "neverOpen",
+            "port": 9229,
+          }
+    ]
+}

packages/override-tools/babel.config.js

@@ -0,0 +1,14 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ * @ts-check
+ */
+
+module.exports = {
+  presets: [
+    ['@babel/preset-env', {targets: {node: 'current'}}],
+    '@babel/preset-typescript',
+  ],
+};

packages/override-tools/jest.config.js

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ * @ts-check
+ */
+
+// For a detailed explanation regarding each configuration property, visit:
+// https://jestjs.io/docs/en/configuration.html
+
+module.exports = {
+  // A list of paths to directories that Jest should use to search for files in
+  roots: ['<rootDir>/src/'],
+
+  // The test environment that will be used for testing
+  testEnvironment: 'node',
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  transformIgnorePatterns: ['/node_modules/(?!io-ts/*|fp-ts/*)'],
+};

packages/override-tools/just-task.js

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ * @ts-check
+ */
+
+const {eslintTask, series, task, taskPresets} = require('just-scripts');
+
+taskPresets.lib();
+
+task('eslint', () => {
+  return eslintTask();
+});
+task('lint', series('eslint'));

packages/override-tools/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "react-native-windows-override-tools",
+  "version": "0.0.1",
+  "description": "Tooling to manage Javascript file overrides in React Native Windows",
+  "private": true,
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "[email protected]:microsoft/react-native-windows.git",
+    "directory": "packages/override-tools"
+  },
+  "scripts": {
+    "build": "just-scripts build",
+    "clean": "just-scripts clean",
+    "lint": "just-scripts lint",
+    "lint:fix": "eslint ./**/*.ts --fix",
+    "test": "just-scripts test",
+    "watch": "tsc -w"
+  },
+  "dependencies": {
+    "io-ts": "^2.1.1",
+    "lodash": "^4.17.15",
+    "ora": "^4.0.3",
+    "simple-git": "^1.131.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.8.4",
+    "@babel/preset-env": "^7.8.4",
+    "@babel/preset-typescript": "^7.8.3",
+    "@types/jest": "^24.9.1",
+    "@types/lodash": "^4.14.149",
+    "@types/node": "^13.7.4",
+    "@types/ora": "^3.2.0",
+    "babel-jest": "^24.9.0",
+    "fp-ts": "^2.5.0",
+    "jest": "^24.9.0"
+  }
+}

packages/override-tools/src/FileRepository.ts

@@ -0,0 +1,51 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ */
+
+/**
+ * Provides access to patch files
+ */
+export interface OverrideFileRepository {
+  /**
+   * Return the repository-relative path to all patch files
+   */
+  listFiles(): Promise<Array<string>>;
+
+  /**
+   * Read the contents of a patch file
+   */
+  getFileContents(filename: string): Promise<string | null>;
+}
+
+/**
+ * Provides access to React Native source files
+ */
+export interface ReactFileRepository {
+  getFileContents(filename: string): Promise<string | null>;
+}
+
+/**
+ * Provides access to React Native source files of arbitrary version
+ */
+export interface VersionedReactFileRepository {
+  getFileContents(
+    filename: string,
+    reactNativeVersion: string,
+  ): Promise<string | null>;
+}
+
+/**
+ * Convert from a VersionedReactFileRepository to ReactFileRepository
+ */
+export function bindVersion(
+  repository: VersionedReactFileRepository,
+  version: string,
+): ReactFileRepository {
+  return {
+    getFileContents: (filename: string) =>
+      repository.getFileContents(filename, version),
+  };
+}

packages/override-tools/src/GitReactFileRepository.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ */
+
+import * as fs from './fs-promise';
+import * as path from 'path';
+import * as simplegit from 'simple-git/promise';
+
+import {VersionedReactFileRepository} from './FileRepository';
+
+const REACT_NATIVE_GITHUB_URL = 'https://github.com/facebook/react-native.git';
+
+/**
+ * Retrives React Native files using the React Native Github repo. Switching
+ * between getting file contents of different versions may be slow.
+ */
+export class GitReactFileRepository implements VersionedReactFileRepository {
+  private gitClient: simplegit.SimpleGit;
+  private gitDirectory: string;
+
+  private constructor() {}
+
+  static async createAndInit(
+    gitDirectory: string,
+  ): Promise<GitReactFileRepository> {
+    let reactFileRepo = new GitReactFileRepository();
+    reactFileRepo.gitDirectory = gitDirectory;
+
+    await fs.mkdir(gitDirectory, {recursive: true});
+
+    const gitClient = (reactFileRepo.gitClient = simplegit(gitDirectory));
+    if (await gitClient.checkIsRepo()) {
+      await gitClient.fetch();
+    } else {
+      await gitClient.clone(REACT_NATIVE_GITHUB_URL, gitDirectory);
+    }
+
+    return reactFileRepo;
+  }
+
+  async getFileContents(
+    filename: string,
+    reactNativeVersion: string,
+  ): Promise<string | null> {
+    await this.gitClient.checkout(`v${reactNativeVersion}`);
+
+    const filePath = path.join(this.gitDirectory, filename);
+    if (await fs.exists(filePath)) {
+      return (await fs.readFile(filePath)).toString();
+    } else {
+      return null;
+    }
+  }
+}

packages/override-tools/src/Manifest.ts

@@ -0,0 +1,135 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ */
+
+import * as _ from 'lodash';
+import * as crypto from 'crypto';
+import * as fs from './fs-promise';
+import * as t from 'io-ts';
+
+import {OverrideFileRepository, ReactFileRepository} from './FileRepository';
+
+import {ThrowReporter} from 'io-ts/es6/ThrowReporter';
+
+/**
+ * Manifest entry type class for "platform" overrides. I.e. overrides not
+ * patching shared code, or derived from existing code.
+ */
+const PlatformEntryType = t.type({
+  type: t.literal('platform'),
+  file: t.string,
+});
+
+/**
+ * Manifest entry type class for overrides that derive or patch from upstream
+ * code.
+ */
+const NonPlatformEntryType = t.type({
+  type: t.union([t.literal('patch'), t.literal('derived')]),
+  file: t.string,
+  baseFile: t.string,
+  baseVersion: t.string,
+  baseHash: t.string,
+
+  // Allow LEGACY_FIXME for existing overrides that don't have issues yet
+  issue: t.union([t.number, t.literal('LEGACY_FIXME')]),
+});
+
+const EntryType = t.union([PlatformEntryType, NonPlatformEntryType]);
+const ManifestType = t.type({overrides: t.array(EntryType)});
+
+export type PlatformEntry = t.TypeOf<typeof PlatformEntryType>;
+export type NonPlatformEntry = t.TypeOf<typeof NonPlatformEntryType>;
+export type Entry = t.TypeOf<typeof EntryType>;
+export type Manifest = t.TypeOf<typeof ManifestType>;
+
+/**
+ * Read an override manifest from a file.
+ *
+ * @throws if the file is invalid or cannot be found
+ */
+export async function readFromFile(filePath: string): Promise<Manifest> {
+  const json = (await fs.readFile(filePath)).toString();
+  return this.parse(json);
+}
+
+/**
+ * Parse a string with JSON for the override manifest into one.
+ *
+ * @throws if the JSON doesn't describe a valid manifest
+ */
+export function parse(json: string): Manifest {
+  const parsed = JSON.parse(json);
+
+  ThrowReporter.report(ManifestType.decode(parsed));
+  return parsed;
+}
+
+export interface ValidationError {
+  type:
+    | 'fileMissingFromManifest' // An override file is present with no manifest entry
+    | 'overrideFileNotFound' // The manifest describes a file which does not exist
+    | 'baseFileNotFound' // The base file for a manifest entry cannot be found
+    | 'outOfDate'; // A base file has changed since the manifested version
+  file: string;
+}
+
+/**
+ * Check that overrides are accurately accounted for in the manifest. I.e. we
+ * should have a 1:1 mapping between files and manifest entries, and base files
+ * should be present and unchanged since entry creation.
+ */
+export async function validate(
+  manifest: Manifest,
+  overrideRepo: OverrideFileRepository,
+  reactRepo: ReactFileRepository,
+): Promise<Array<ValidationError>> {
+  const errors: Array<ValidationError> = [];
+
+  const manifestedFiles = manifest.overrides.map(override => override.file);
+  const overrideFiles = await overrideRepo.listFiles();
+
+  const fileMissingFromManifest = _.difference(overrideFiles, manifestedFiles);
+  fileMissingFromManifest.forEach(file =>
+    errors.push({type: 'fileMissingFromManifest', file: file}),
+  );
+  const overridesNotFound = _.difference(manifestedFiles, overrideFiles);
+  overridesNotFound.forEach(file =>
+    errors.push({type: 'overrideFileNotFound', file: file}),
+  );
+
+  await Promise.all(
+    manifest.overrides.map(async override => {
+      if (override.type === 'platform') {
+        return;
+      }
+
+      const baseContent = await reactRepo.getFileContents(override.baseFile);
+      if (baseContent === null) {
+        errors.push({type: 'baseFileNotFound', file: override.baseFile});
+        return;
+      }
+
+      const baseHash = hashContent(baseContent);
+      if (baseHash.toLowerCase() !== override.baseHash.toLowerCase()) {
+        errors.push({type: 'outOfDate', file: override.file});
+        return;
+      }
+    }),
+  );
+
+  return errors;
+}
+
+/**
+ * Hash content into the form expected in a manifest entry. Exposed for
+ * testing.
+ */
+export function hashContent(str: string) {
+  const hasher = crypto.createHash('sha1');
+  hasher.update(str);
+  return hasher.digest('hex');
+}