Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.x] Bundle Node 14 as a fallback for operating systems that cannot run Node 18 #4175

Merged
merged 1 commit into from
May 30, 2023
Merged

[Backport 2.x] Bundle Node 14 as a fallback for operating systems that cannot run Node 18 #4175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions scripts/use_node
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,17 @@ elif [ ! -z "$NODE_HOME" ]; then
NODE_ERROR_MSG="in NODE_HOME"
NODE_ERROR_SHOW=true
else
# Set these variables outside, as catchalls, to show meaningful errors if needed
NODE="$OSD_HOME/node/bin/node"
NODE_ERROR_MSG="bundled with OpenSearch Dashboards"
# A bin folder at the root is only present in release builds that have a bundled Node.js binary
if [ -x "OSD_HOME/bin" ]; then
if [ -d "${OSD_HOME}/bin" ]; then
NODE_ERROR_SHOW=true
# Not all operating systems can run the latest Node.js and the fallback is for them
"${NODE}" -v 2> /dev/null
if [ $? -ne 0 ] && [ -d "${OSD_HOME}/node/fallback" ]; then
NODE="$OSD_HOME/node/fallback/bin/node"
fi
fi
fi

Expand All @@ -75,7 +81,7 @@ else
fi

if [ ! -x "$NODE" ]; then
# Irrespective of NODE_ERROR_SHOW, show the error
# Irrespective of NODE_ERROR_SHOW, if NODE is not found or executable, show the error
echo "Could not find a Node.js runtime binary $NODE_ERROR_MSG or on the system" >&2
exit 1
fi
Expand Down
16 changes: 15 additions & 1 deletion src/dev/build/tasks/create_archives_sources_task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
*/

import { scanCopy, Task } from '../lib';
import { getNodeDownloadInfo } from './nodejs';
import { getNodeDownloadInfo, getNodeVersionDownloadInfo, NODE14_FALLBACK_VERSION } from './nodejs';

export const CreateArchivesSources: Task = {
description: 'Creating platform-specific archive source directories',
Expand All @@ -54,6 +54,20 @@ export const CreateArchivesSources: Task = {
destination: build.resolvePathForPlatform(platform, 'node'),
});

// ToDo [NODE14]: Remove this Node.js 14 fallback download
// Copy the Node.js 14 binaries into node/fallback to be used by `use_node`
await scanCopy({
source: (
await getNodeVersionDownloadInfo(
NODE14_FALLBACK_VERSION,
platform.getNodeArch(),
platform.isWindows(),
config.resolveFromRepo()
)
).extractDir,
destination: build.resolvePathForPlatform(platform, 'node', 'fallback'),
});

log.debug('Node.js copied into', platform.getNodeArch(), 'specific build directory');
})
);
Expand Down
50 changes: 49 additions & 1 deletion src/dev/build/tasks/nodejs/download_node_builds_task.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ jest.mock('../../lib/get_build_number');

expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog));

const { getNodeDownloadInfo } = jest.requireMock('./node_download_info');
const { getNodeDownloadInfo, getNodeVersionDownloadInfo } = jest.requireMock(
'./node_download_info'
);
const { getNodeShasums } = jest.requireMock('./node_shasums');
const { download } = jest.requireMock('../../lib/download');

Expand Down Expand Up @@ -76,6 +78,16 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
};
});

getNodeVersionDownloadInfo.mockImplementation((version, architecture, isWindows, repoRoot) => {
return {
url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`,
downloadName: `node-v${version}-${architecture}.tar.gz`,
downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`,
extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`,
version,
};
});

getNodeShasums.mockReturnValue({
'linux:downloadName': 'linux:sha256',
'darwin:downloadName': 'darwin:sha256',
Expand Down Expand Up @@ -134,6 +146,42 @@ it('downloads node builds for each platform', async () => {
"url": "win32:url",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-linux-x64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-linux-x64.tar.gz",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-linux-arm64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-linux-arm64.tar.gz",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-darwin-x64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-darwin-x64.tar.gz",
},
],
Array [
Object {
"destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-win32-x64.tar.gz",
"log": <ToolingLog>,
"retries": 3,
"sha256": undefined,
"url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-win32-x64.tar.gz",
},
],
]
`);
expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
Expand Down
35 changes: 30 additions & 5 deletions src/dev/build/tasks/nodejs/download_node_builds_task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,25 @@

import { download, GlobalTask } from '../../lib';
import { getNodeShasums } from './node_shasums';
import { getNodeDownloadInfo, getRequiredVersion } from './node_download_info';
import {
getNodeDownloadInfo,
getNodeVersionDownloadInfo,
getRequiredVersion,
NODE14_FALLBACK_VERSION,
} from './node_download_info';

export const DownloadNodeBuilds: GlobalTask = {
global: true,
description: 'Downloading node.js builds for all platforms',
async run(config, log) {
const requiredNodeVersion = getRequiredVersion(config);
const shasums = await getNodeShasums(log, requiredNodeVersion);
await Promise.all(
config.getTargetPlatforms().map(async (platform) => {

// ToDo [NODE14]: Remove this Node.js 14 fallback download
const node14ShaSums = await getNodeShasums(log, NODE14_FALLBACK_VERSION);

await Promise.all([
...config.getTargetPlatforms().map(async (platform) => {
const { url, downloadPath, downloadName } = await getNodeDownloadInfo(config, platform);
await download({
log,
Expand All @@ -48,7 +57,23 @@ export const DownloadNodeBuilds: GlobalTask = {
destination: downloadPath,
retries: 3,
});
})
);
}),
// ToDo [NODE14]: Remove this Node.js 14 fallback download
...config.getTargetPlatforms().map(async (platform) => {
const { url, downloadPath, downloadName } = await getNodeVersionDownloadInfo(
NODE14_FALLBACK_VERSION,
platform.getNodeArch(),
platform.isWindows(),
config.resolveFromRepo()
);
await download({
log,
url,
sha256: node14ShaSums[downloadName],
destination: downloadPath,
retries: 3,
});
}),
]);
},
};
28 changes: 28 additions & 0 deletions src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,27 @@ it('runs expected fs operations', async () => {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-linux-x64.tar.gz,
<absolute path>/.node_binaries/14.21.3/linux-x64,
Object {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-linux-arm64.tar.gz,
<absolute path>/.node_binaries/14.21.3/linux-arm64,
Object {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-darwin-x64.tar.gz,
<absolute path>/.node_binaries/14.21.3/darwin-x64,
Object {
"strip": 1,
},
],
],
"unzip": Array [
Array [
Expand All @@ -132,6 +153,13 @@ it('runs expected fs operations', async () => {
"strip": 1,
},
],
Array [
<absolute path>/.node_binaries/14.21.3/node-v14.21.3-win-x64.zip,
<absolute path>/.node_binaries/14.21.3/win32-x64,
Object {
"strip": 1,
},
],
],
}
`);
Expand Down
28 changes: 23 additions & 5 deletions src/dev/build/tasks/nodejs/extract_node_builds_task.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,39 @@
*/

import { untar, unzip, GlobalTask } from '../../lib';
import { getNodeDownloadInfo } from './node_download_info';
import {
getNodeDownloadInfo,
getNodeVersionDownloadInfo,
NODE14_FALLBACK_VERSION,
} from './node_download_info';

export const ExtractNodeBuilds: GlobalTask = {
global: true,
description: 'Extracting node.js builds for all platforms',
async run(config) {
await Promise.all(
config.getTargetPlatforms().map(async (platform) => {
await Promise.all([
...config.getTargetPlatforms().map(async (platform) => {
const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform);
if (platform.isWindows()) {
await unzip(downloadPath, extractDir, { strip: 1 });
} else {
await untar(downloadPath, extractDir, { strip: 1 });
}
})
);
}),
// ToDo [NODE14]: Remove this Node.js 14 fallback download
...config.getTargetPlatforms().map(async (platform) => {
const { downloadPath, extractDir } = await getNodeVersionDownloadInfo(
NODE14_FALLBACK_VERSION,
platform.getNodeArch(),
platform.isWindows(),
config.resolveFromRepo()
);
if (platform.isWindows()) {
await unzip(downloadPath, extractDir, { strip: 1 });
} else {
await untar(downloadPath, extractDir, { strip: 1 });
}
}),
]);
},
};
27 changes: 26 additions & 1 deletion src/dev/build/tasks/nodejs/node_download_info.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@
* under the License.
*/

import { basename } from 'path';
import { basename, resolve } from 'path';
import fetch from 'node-fetch';
import semver from 'semver';

import { Config, Platform } from '../../lib';

const NODE_RANGE_CACHE: { [key: string]: string } = {};

export const NODE14_FALLBACK_VERSION = '14.21.3';

export async function getNodeDownloadInfo(config: Config, platform: Platform) {
const version = getRequiredVersion(config);
const arch = platform.getNodeArch();
Expand All @@ -57,6 +59,29 @@ export async function getNodeDownloadInfo(config: Config, platform: Platform) {
};
}

export async function getNodeVersionDownloadInfo(
version: string,
architecture: string,
isWindows: boolean,
repoRoot: string
) {
const downloadName = isWindows
? `node-v${version}-win-x64.zip`
: `node-v${version}-${architecture}.tar.gz`;

const url = `https://mirrors.nodejs.org/dist/v${version}/${downloadName}`;
const downloadPath = resolve(repoRoot, '.node_binaries', version, basename(downloadName));
const extractDir = resolve(repoRoot, '.node_binaries', version, architecture);

return {
url,
downloadName,
downloadPath,
extractDir,
version,
};
}

export async function getLatestNodeVersion(config: Config) {
const range = config.getNodeRange();
// Check cache and return if known
Expand Down