Skip to content

Commit

Permalink
Support http proxies
Browse files Browse the repository at this point in the history
  • Loading branch information
msujew committed Mar 30, 2022
1 parent 7b8d3b7 commit 68bb515
Show file tree
Hide file tree
Showing 45 changed files with 1,296 additions and 68 deletions.
4 changes: 1 addition & 3 deletions dev-packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,15 @@
"@theia/ffmpeg": "1.23.0",
"@theia/localization-manager": "1.23.0",
"@theia/ovsx-client": "1.23.0",
"@theia/request-service": "1.23.0",
"@types/chai": "^4.2.7",
"@types/mocha": "^5.2.7",
"@types/node-fetch": "^2.5.7",
"@types/puppeteer": "^2.0.0",
"chai": "^4.2.0",
"chalk": "4.0.0",
"decompress": "^4.2.1",
"https-proxy-agent": "^5.0.0",
"mocha": "^7.0.0",
"node-fetch": "^2.6.7",
"proxy-from-env": "^1.1.0",
"puppeteer": "^2.0.0",
"puppeteer-to-istanbul": "^1.2.2",
"temp": "^0.9.1",
Expand Down
66 changes: 33 additions & 33 deletions dev-packages/cli/src/download-plugins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,12 @@ declare global {
import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
import * as chalk from 'chalk';
import * as decompress from 'decompress';
import { createWriteStream, promises as fs } from 'fs';
import { HttpsProxyAgent } from 'https-proxy-agent';
import fetch, { RequestInit, Response } from 'node-fetch';
import { promises as fs } from 'fs';
import * as path from 'path';
import { getProxyForUrl } from 'proxy-from-env';
import * as stream from 'stream';
import * as temp from 'temp';
import { promisify } from 'util';
import { NodeRequestService } from '@theia/request-service/lib/node-request-service';
import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';

const pipelineAsPromised = promisify(stream.pipeline);
import { RequestContext } from '@theia/request-service';

temp.track();

Expand Down Expand Up @@ -66,16 +61,31 @@ export interface DownloadPluginsOptions {
* The open-vsx registry API url.
*/
apiUrl?: string;

proxyUrl?: string;
proxyAuthorization?: string;
strictSsl?: boolean;
}

const requestService = new NodeRequestService();

export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise<void> {
const {
packed = false,
ignoreErrors = false,
apiVersion = DEFAULT_SUPPORTED_API_VERSION,
apiUrl = 'https://open-vsx.org/api'
apiUrl = 'https://open-vsx.org/api',
proxyUrl,
proxyAuthorization,
strictSsl
} = options;

requestService.configure({
proxyUrl,
proxyAuthorization,
strictSSL: strictSsl
});

// Collect the list of failures to be appended at the end of the script.
const failures: string[] = [];

Expand Down Expand Up @@ -111,7 +121,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
const extensionPacks = await collectExtensionPacks(pluginsDir, excludedIds);
if (extensionPacks.size > 0) {
console.warn(`--- resolving ${extensionPacks.size} extension-packs ---`);
const client = new OVSXClient({ apiVersion, apiUrl });
const client = new OVSXClient({ apiVersion, apiUrl }, requestService);
// De-duplicate extension ids to only download each once:
const ids = new Set<string>(Array.from(extensionPacks.values()).flat());
await Promise.all(Array.from(ids, async id => {
Expand All @@ -127,7 +137,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
const pluginDependencies = await collectPluginDependencies(pluginsDir, excludedIds);
if (pluginDependencies.length > 0) {
console.warn(`--- resolving ${pluginDependencies.length} extension dependencies ---`);
const client = new OVSXClient({ apiVersion, apiUrl });
const client = new OVSXClient({ apiVersion, apiUrl }, requestService);
// De-duplicate extension ids to only download each once:
const ids = new Set<string>(pluginDependencies);
await Promise.all(Array.from(ids, async id => {
Expand Down Expand Up @@ -187,20 +197,23 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl

let attempts: number;
let lastError: Error | undefined;
let response: Response | undefined;
let response: RequestContext | undefined;

for (attempts = 0; attempts < maxAttempts; attempts++) {
if (attempts > 0) {
await new Promise(resolve => setTimeout(resolve, retryDelay));
}
lastError = undefined;
try {
response = await xfetch(pluginUrl);
response = await requestService.request({
url: pluginUrl
});
} catch (error) {
lastError = error;
continue;
}
const retry = response.status === 439 || response.status >= 500;
const status = response.res.statusCode;
const retry = status && (status === 439 || status >= 500);
if (!retry) {
break;
}
Expand All @@ -213,20 +226,19 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
failures.push(chalk.red(`x ${plugin}: failed to download (unknown reason)`));
return;
}
if (response.status !== 200) {
failures.push(chalk.red(`x ${plugin}: failed to download with: ${response.status} ${response.statusText}`));
if (response.res.statusCode !== 200) {
failures.push(chalk.red(`x ${plugin}: failed to download with: ${response.res.statusCode}`));
return;
}

if ((fileExt === '.vsix' || fileExt === '.theia') && packed === true) {
// Download .vsix without decompressing.
const file = createWriteStream(targetPath);
await pipelineAsPromised(response.body, file);
await fs.writeFile(targetPath, response.buffer.buffer);
} else {
await fs.mkdir(targetPath, { recursive: true });
const tempFile = temp.createWriteStream('theia-plugin-download');
await pipelineAsPromised(response.body, tempFile);
await decompress(tempFile.path, targetPath);
const tempFile = temp.path('theia-plugin-download');
await fs.writeFile(tempFile, Buffer.from(response.buffer.buffer));
await decompress(tempFile, targetPath);
}

console.warn(chalk.green(`+ ${plugin}${version ? `@${version}` : ''}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
Expand All @@ -242,18 +254,6 @@ async function isDownloaded(filePath: string): Promise<boolean> {
return fs.stat(filePath).then(() => true, () => false);
}

/**
* Follow HTTP(S)_PROXY, ALL_PROXY and NO_PROXY environment variables.
*/
export function xfetch(url: string, options?: RequestInit): Promise<Response> {
const proxiedOptions: RequestInit = { ...options };
const proxy = getProxyForUrl(url);
if (!proxiedOptions.agent && proxy !== '') {
proxiedOptions.agent = new HttpsProxyAgent(proxy);
}
return fetch(url, proxiedOptions);
}

/**
* Walk the plugin directory and collect available extension paths.
* @param pluginDir the plugin directory.
Expand Down
23 changes: 20 additions & 3 deletions dev-packages/cli/src/theia.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,13 @@ async function theiaCli(): Promise<void> {
}
})
.command<{
packed: boolean
packed: boolean,
ignoreErrors: boolean,
apiVersion?: string,
apiUrl?: string,
proxyUrl?: string,
proxyAuthentification?: string,
strictSsl: boolean
}>({
command: 'download:plugins',
describe: 'Download defined external plugins',
Expand All @@ -223,10 +229,21 @@ async function theiaCli(): Promise<void> {
alias: 'u',
describe: 'Open-VSX Registry API URL',
default: 'https://open-vsx.org/api'
},
'proxy-url': {
describe: 'Proxy URL'
},
'proxy-authentification': {
describe: 'Proxy authentification information'
},
'strict-ssl': {
describe: 'Whether to enable strict SSL mode',
boolean: true,
default: false
}
},
handler: async ({ packed }) => {
await downloadPlugins({ packed });
handler: async args => {
await downloadPlugins(args);
},
})
.command<{
Expand Down
3 changes: 3 additions & 0 deletions dev-packages/cli/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
},
{
"path": "../ovsx-client"
},
{
"path": "../request-service"
}
]
}
3 changes: 1 addition & 2 deletions dev-packages/ovsx-client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@
"watch": "theiaext watch"
},
"dependencies": {
"@types/bent": "^7.0.1",
"bent": "^7.1.0",
"@theia/request-service": "1.23.0",
"semver": "^5.4.1"
}
}
3 changes: 2 additions & 1 deletion dev-packages/ovsx-client/src/ovsx-client.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import * as chai from 'chai';
import { OVSXClient } from './ovsx-client';
import { NodeRequestService } from '@theia/request-service/lib/node-request-service';
import { VSXSearchParam } from './ovsx-types';

const expect = chai.expect;
Expand All @@ -31,7 +32,7 @@ describe('OVSX Client', () => {
client = new OVSXClient({
apiVersion,
apiUrl
});
}, new NodeRequestService());
});

describe('isEngineValid', () => {
Expand Down
32 changes: 22 additions & 10 deletions dev-packages/ovsx-client/src/ovsx-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
// *****************************************************************************

import * as bent from 'bent';
import * as semver from 'semver';
import {
VSXAllVersions,
Expand All @@ -26,9 +25,7 @@ import {
VSXSearchParam,
VSXSearchResult
} from './ovsx-types';

const fetchText = bent('GET', 'string', 200);
const fetchJson = bent('GET', { 'Accept': 'application/json' }, 'json', 200);
import { RequestContext, RequestService } from '@theia/request-service';

export interface OVSXClientOptions {
apiVersion: string
Expand All @@ -37,11 +34,19 @@ export interface OVSXClientOptions {

export class OVSXClient {

constructor(readonly options: OVSXClientOptions) { }
constructor(readonly options: OVSXClientOptions, readonly request: RequestService) { }

async search(param?: VSXSearchParam): Promise<VSXSearchResult> {
const searchUri = await this.buildSearchUri(param);
return this.fetchJson<VSXSearchResult>(searchUri);
try {
return await this.fetchJson<VSXSearchResult>(searchUri);
} catch (err) {
return {
error: err?.message || String(err),
offset: 0,
extensions: []
};
}
}

protected async buildSearchUri(param?: VSXSearchParam): Promise<string> {
Expand Down Expand Up @@ -99,12 +104,19 @@ export class OVSXClient {
throw new Error(`Extension with id ${id} not found at ${apiUri}`);
}

protected fetchJson<R>(url: string): Promise<R> {
return fetchJson(url) as Promise<R>;
protected async fetchJson<R>(url: string): Promise<R> {
const e = await this.request.request({
url,
headers: { 'Accept': 'application/json' }
});
return RequestContext.asJson<R>(e);
}

fetchText(url: string): Promise<string> {
return fetchText(url);
async fetchText(url: string): Promise<string> {
const e = await this.request.request({
url
});
return RequestContext.asText(e);
}

/**
Expand Down
6 changes: 5 additions & 1 deletion dev-packages/ovsx-client/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,9 @@
"include": [
"src"
],
"references": []
"references": [
{
"path": "../request-service"
}
]
}
10 changes: 10 additions & 0 deletions dev-packages/request-service/.eslintrc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/** @type {import('eslint').Linter.Config} */
module.exports = {
extends: [
'../../configs/build.eslintrc.json'
],
parserOptions: {
tsconfigRootDir: __dirname,
project: 'tsconfig.json'
}
};
29 changes: 29 additions & 0 deletions dev-packages/request-service/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<div align='center'>

<br />

<img src='https://raw.githubusercontent.com/eclipse-theia/theia/master/logo/theia.svg?sanitize=true' alt='theia-ext-logo' width='100px' />

<h2>ECLIPSE THEIA - REQUEST SERVICE</h2>

<hr />

</div>

## Description

The `@theia/request-service` package is used to send proxy-aware http requests to other service.

## Additional Information

- [Theia - GitHub](https://github.com/eclipse-theia/theia)
- [Theia - Website](https://theia-ide.org/)

## License

- [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/)
- [一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp)

## Trademark
"Theia" is a trademark of the Eclipse Foundation
https://www.eclipse.org/theia
36 changes: 36 additions & 0 deletions dev-packages/request-service/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"name": "@theia/request-service",
"version": "1.23.0",
"description": "Theia Proxy-Aware Request Service",
"publishConfig": {
"access": "public"
},
"license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
"repository": {
"type": "git",
"url": "https://github.com/eclipse-theia/theia.git"
},
"bugs": {
"url": "https://github.com/eclipse-theia/theia/issues"
},
"homepage": "https://github.com/eclipse-theia/theia",
"files": [
"lib",
"src"
],
"main": "lib/index.js",
"typings": "lib/index.d.ts",
"scripts": {
"build": "theiaext build",
"clean": "theiaext clean",
"compile": "theiaext compile",
"lint": "theiaext lint",
"test": "theiaext test",
"watch": "theiaext watch"
},
"dependencies": {
"http-proxy-agent": "^5.0.0",
"https-proxy-agent": "^5.0.0",
"vscode-languageserver-protocol": "~3.15.3"
}
}
Loading

0 comments on commit 68bb515

Please sign in to comment.