[remote] Support download of native dependencies

.github/workflows/native-dependency-build.yml

@@ -0,0 +1,28 @@
+name: Build-Native-Dependencies
+
+on: workflow_dispatch
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        node-version: [16.x]
+        os: ['ubuntu-latest', 'windows-latest', 'macos-latest']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+    - run: npm ci
+    - run: yarn prebuild:native-dependencies
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v3 
+      with: 
+        name: native-dependencies
+        path: ./prebuild-native-dependencies/*.zip

.gitignore

@@ -33,3 +33,4 @@ dependency-check-summary.txt*
 *-trace.json
 .tours
 /performance-result.json
+prebuild-native-dependencies

doc/Publishing.md

@@ -3,16 +3,21 @@
 
 ## Release Procedure
 
-- [**Yarn Upgrade**](#yarn-upgrade)
-- [**Announce Release**](#announce-release)
-- [**Pre-Publishing Steps**](#pre-publishing-steps)
-- [**Login to NPM Registry**](#login-to-npm-registry)
-- [**Publish Packages**](#publish-packages)
-- [**Commit Updated Versions**](#commit-updated-version)
-- [**Create Release**](#create-release)
-- [**Update Eclipse Release Page**](#update-eclipse-release-page)
-- [**Post-Release**](#post-release)
+- [Publishing](#publishing)
+  - [Release Procedure](#release-procedure)
+  - [Yarn Upgrade](#yarn-upgrade)
+  - [Announce Release](#announce-release)
+  - [Pre-Release Steps](#pre-release-steps)
+  - [Pre-Publishing Steps](#pre-publishing-steps)
+  - [Login to NPM Registry](#login-to-npm-registry)
+  - [Publish Packages](#publish-packages)
+  - [Commit Updated Version](#commit-updated-version)
+  - [Create Release](#create-release)
+  - [Update Eclipse Release Page](#update-eclipse-release-page)
+  - [Post Release](#post-release)
     - [Announce Release is Completed](#announce-release-is-completed)
+    - [Update Milestones](#update-milestones)
+    - [Update Roadmap](#update-roadmap)
 
 ## Yarn Upgrade
 
@@ -150,6 +155,11 @@ Login to [Eclipse Foundation Theia project page]( https://projects.eclipse.org/p
 
 ## Post Release
 
+Upload prebuild native dependencies to the release:
+- Execute the `Build-Native-Dependencies` Github Action
+- Download the `native-dependencies` artifact from the Job
+- Upload the contents of the artifact as release assets to the theia release
+
 ### Announce Release is Completed
 
 - Update the forum release post to announce that the release has completed.

examples/browser/package.json

@@ -44,6 +44,7 @@
     "@theia/preview": "1.38.0",
     "@theia/process": "1.38.0",
     "@theia/property-view": "1.38.0",
+    "@theia/remote": "1.38.0",
     "@theia/scm": "1.38.0",
     "@theia/scm-extra": "1.38.0",
     "@theia/search-in-workspace": "1.38.0",

examples/browser/tsconfig.json

@@ -95,6 +95,9 @@
     {
       "path": "../../packages/property-view"
     },
+    {
+      "path": "../../packages/remote"
+    },
     {
       "path": "../../packages/scm"
     },

examples/electron/package.json

@@ -45,6 +45,7 @@
     "@theia/preview": "1.38.0",
     "@theia/process": "1.38.0",
     "@theia/property-view": "1.38.0",
+    "@theia/remote": "1.38.0",
     "@theia/scm": "1.38.0",
     "@theia/scm-extra": "1.38.0",
     "@theia/search-in-workspace": "1.38.0",

examples/electron/tsconfig.json

@@ -98,6 +98,9 @@
     {
       "path": "../../packages/property-view"
     },
+    {
+      "path": "../../packages/remote"
+    },
     {
       "path": "../../packages/scm"
     },

package.json

@@ -21,6 +21,7 @@
     "@typescript-eslint/eslint-plugin": "^4.8.1",
     "@typescript-eslint/eslint-plugin-tslint": "^4.8.1",
     "@typescript-eslint/parser": "^4.8.1",
+    "archiver": "5.3.1",
     "chai": "4.3.4",
     "chai-spies": "1.0.0",
     "chai-string": "^1.4.0",
@@ -78,6 +79,7 @@
     "lint:oneshot": "node --max-old-space-size=4096 node_modules/eslint/bin/eslint.js --cache=true \"{dev-packages,packages,examples}/**/*.{ts,tsx}\"",
     "preinstall": "node-gyp install",
     "prepare": "yarn -s compile:references && lerna run prepare && yarn -s compile",
+    "prebuild:native-dependencies": "node scripts/build-native-dependencies",
     "publish:latest": "lerna publish --exact --yes --no-push && yarn -s publish:check",
     "publish:next": "lerna publish preminor --exact --canary --preid next --dist-tag next --no-git-reset --no-git-tag-version --no-push --yes && yarn -s publish:check",
     "publish:check": "node scripts/check-publish.js",

packages/core/src/node/backend-application-module.ts

@@ -41,6 +41,8 @@ import { bindNodeStopwatch, bindBackendStopwatchServer } from './performance';
 import { OSBackendApplicationContribution } from './os-backend-application-contribution';
 import { BackendRequestFacade } from './request/backend-request-facade';
 import { FileSystemLocking, FileSystemLockingImpl } from './filesystem-locking';
+import { DependencyDownloadContribution, DependencyDownloadService, DummyDependencyDownloader } from './dependency-download';
+import { DrivelistDependencyDownload, keytarDependencyDownload, NSFWDependencyDownload } from './native-dependency-download-contributions';
 
 decorate(injectable(), ApplicationPackage);
 
@@ -131,4 +133,10 @@ export const backendApplicationModule = new ContainerModule(bind => {
     bindBackendStopwatchServer(bind);
 
     bind(FileSystemLocking).to(FileSystemLockingImpl).inSingletonScope();
+    bind(DependencyDownloadService).to(DummyDependencyDownloader);
+    bindContributionProvider(bind, DependencyDownloadContribution.Contribution);
+
+    bind(DependencyDownloadContribution.Contribution).to(keytarDependencyDownload);
+    bind(DependencyDownloadContribution.Contribution).to(DrivelistDependencyDownload);
+    bind(DependencyDownloadContribution.Contribution).to(NSFWDependencyDownload);
 });

packages/core/src/node/dependency-download.ts

@@ -0,0 +1,87 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { NodeRequestOptions } from '@theia/request/lib/node-request-service';
+import { MaybePromise } from 'src/common';
+export interface FileDependencyResult {
+    targetFile: string;
+    /** Some files needs to be made executable on the target system */
+    chmod?: number;
+}
+
+export interface FileDependencyDownload {
+    /** Defaults to `true` */
+    unzip?: boolean;
+    file: FileDependencyResult
+    downloadHandler: string | Buffer | (() => Promise<Buffer | string>)
+}
+
+// Directories are expected to be in a zipped format anyway
+// We always unzip them and call `files` on each contained file
+export interface DirectoryDependencyDownload {
+    files: (path: string) => FileDependencyResult;
+    downloadHandler: string | Buffer | (() => Promise<Buffer | string>)
+}
+
+export interface DownloadOptions {
+    remoteOS: string;
+    theiaVersion: string;
+    /**
+     * These are the `NodeRequestOptions` for the `NodeRequestService`
+     * returns undefined when url from requestInfo has been downloaded previously
+     */
+    download: (requestInfo: string | NodeRequestOptions) => Promise<Buffer>
+}
+
+/**
+ * contribution used for downloading prebuild nativ dependency when connecting to a remote machine with a different system
+ */
+export interface DependencyDownloadContribution {
+    // used to filter out multiple contributions downloading the same package
+    dependencyId: string;
+
+    download(options: DownloadOptions): MaybePromise<FileDependencyDownload | DirectoryDependencyDownload>;
+}
+
+export namespace DependencyDownloadContribution {
+    export const Contribution = Symbol('dependencyDownloadContribution');
+
+    const DEFAULT_DEPENDENCY_DOWNLOAD_URL = 'https://github.com/eclipse-theia/theia/releases';
+
+    export function getDefaultURLForFile(dependencyName: string, remoteSystem: string, theiaVersion: string): string {
+        return `${DEFAULT_DEPENDENCY_DOWNLOAD_URL}/${theiaVersion ? `tag/v${theiaVersion}` : 'latest'}/${dependencyName}-${remoteSystem}.zip`;
+    }
+
+}
+
+export const DependencyDownloadService = Symbol('dependencyDownloadService');
+/**
+ * used by the "@theia/remote" package to donwload nativ dependencies for the remote system;
+ */
+export interface DependencyDownloadService {
+
+    /**
+     * downloads natvie dependencies for copying on a remote machine
+     * @param remoteSystem the operating system of the remote machine in format "{platform}-{architecure}"" e.g. "win32-x64"
+     */
+    downloadDependencies(remoteSystem: string): MaybePromise<Array<FileDependencyDownload | DirectoryDependencyDownload>>;
+}
+
+export class DummyDependencyDownloader implements DependencyDownloadService {
+    downloadDependencies(remoteSystem: string): Array<FileDependencyDownload | DirectoryDependencyDownload> {
+        return [];
+    }
+}

packages/core/src/node/native-dependency-download-contributions.ts

@@ -0,0 +1,93 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { injectable } from 'inversify';
+import { DependencyDownloadContribution, DirectoryDependencyDownload, DownloadOptions, FileDependencyDownload } from './dependency-download';
+
+@injectable()
+export class DrivelistDependencyDownload implements DependencyDownloadContribution {
+    dependencyId = 'drivelist';
+
+    async download(options: DownloadOptions): Promise<FileDependencyDownload | DirectoryDependencyDownload> {
+        return {
+            file: {
+                targetFile: 'lib/backend/native/drivelist.node'
+            },
+            unzip: true,
+            downloadHandler: await options.download(DependencyDownloadContribution.getDefaultURLForFile('drivelist', options.remoteOS, options.theiaVersion))
+        };
+    }
+}
+
+@injectable()
+export class keytarDependencyDownload implements DependencyDownloadContribution {
+    dependencyId = 'keytar';
+
+    async download(options: DownloadOptions): Promise<FileDependencyDownload | DirectoryDependencyDownload> {
+        return {
+            file: {
+                targetFile: 'lib/backend/native/keytar.node'
+            },
+            unzip: true,
+            downloadHandler: await options.download(DependencyDownloadContribution.getDefaultURLForFile('keytar', options.remoteOS, options.theiaVersion))
+        };
+    }
+}
+
+@injectable()
+export class NSFWDependencyDownload implements DependencyDownloadContribution {
+    dependencyId = 'nsfw';
+
+    async download(options: DownloadOptions): Promise<FileDependencyDownload | DirectoryDependencyDownload> {
+        return {
+            file: {
+                targetFile: 'lib/backend/native/nsfw.node'
+            },
+            unzip: true,
+            downloadHandler: await options.download(DependencyDownloadContribution.getDefaultURLForFile('nsfw', options.remoteOS, options.theiaVersion))
+        };
+    }
+}
+
+const BASE_URL = 'https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-8/ripgrep-v13.0.0-8';
+@injectable()
+export class RigrepDependencyDownload implements DependencyDownloadContribution {
+    dependencyId = 'rigrep';
+    async download(options: DownloadOptions): Promise<FileDependencyDownload | DirectoryDependencyDownload> {
+        return {
+            file: {
+                targetFile: `lib/backend/native/rg${options.remoteOS.startsWith('win32') ? '.exe' : ''}`
+            },
+            unzip: true,
+            downloadHandler: await options.download(this.getDownloadUrl(options.remoteOS))
+        };
+    }
+
+    getDownloadUrl(remoteOS: string): string {
+        const [platform, architecure] = remoteOS.split('-');
+
+        let transformedPlatform: string;
+        if (remoteOS.includes('darwin')) {
+            transformedPlatform = 'apple-darwin';
+        } else if (remoteOS.includes('win')) {
+            transformedPlatform = 'pc-windows-msvc';
+        } else {
+            transformedPlatform = 'unkown-linux-gnu';
+        }
+
+        return `${BASE_URL}-${architecure === 'x64' ? 'x86_64' : architecure}-${transformedPlatform}.${platform.includes('win') ? 'zip' : 'tar.gz'}`;
+    }
+}

packages/file-search/src/node/file-search-backend-module.ts

@@ -18,6 +18,8 @@ import { ContainerModule } from '@theia/core/shared/inversify';
 import { ConnectionHandler, JsonRpcConnectionHandler } from '@theia/core/lib/common';
 import { FileSearchServiceImpl } from './file-search-service-impl';
 import { fileSearchServicePath, FileSearchService } from '../common/file-search-service';
+import { DependencyDownloadContribution } from '@theia/core/lib/node/dependency-download';
+import { RigrepDependencyDownload } from '@theia/core/lib/node/native-dependency-download-contributions';
 
 export default new ContainerModule(bind => {
 
@@ -27,4 +29,6 @@ export default new ContainerModule(bind => {
             ctx.container.get(FileSearchService)
         )
     ).inSingletonScope();
+
+    bind(DependencyDownloadContribution.Contribution).to(RigrepDependencyDownload);
 });

packages/git/src/node/git-backend-module.ts

@@ -30,6 +30,8 @@ import { GitPromptServer, GitPromptClient, GitPrompt } from '../common/git-promp
 import { DugiteGitPromptServer } from './dugite-git-prompt';
 import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module';
 import { DefaultGitInit, GitInit } from './init/git-init';
+import { DependencyDownloadContribution } from '@theia/core/lib/node/dependency-download';
+import { FindGitRepositoriesDependencyDownload } from './nativ-dependencies/find-git-repositories-download-contribution';
 
 const SINGLE_THREADED = process.argv.indexOf('--no-cluster') !== -1;
 
@@ -77,6 +79,7 @@ export function bindGit(bind: interfaces.Bind, bindingOptions: GitBindingOptions
     bind(DefaultGitInit).toSelf();
     bind(GitInit).toService(DefaultGitInit);
     bind(ConnectionContainerModule).toConstantValue(gitConnectionModule);
+    bind(DependencyDownloadContribution.Contribution).to(FindGitRepositoriesDependencyDownload);
 }
 
 const gitConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService }) => {

packages/git/src/node/nativ-dependencies/find-git-repositories-download-contribution.ts

@@ -0,0 +1,33 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { DependencyDownloadContribution, DirectoryDependencyDownload, DownloadOptions, FileDependencyDownload } from '@theia/core/lib/node/dependency-download';
+import { injectable } from '@theia/core/shared/inversify';
+
+@injectable()
+export class FindGitRepositoriesDependencyDownload implements DependencyDownloadContribution {
+    dependencyId = 'find-git-repositories';
+
+    async download(options: DownloadOptions): Promise<FileDependencyDownload | DirectoryDependencyDownload> {
+        return {
+            file: {
+                targetFile: 'lib/backend/native/findGitRepos.node'
+            },
+            unzip: true,
+            downloadHandler: await options.download(DependencyDownloadContribution.getDefaultURLForFile('findGitRepos', options.remoteOS, options.theiaVersion))
+        };
+    }
+}