diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 144cf5fcd..a52f23690 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -1,6 +1,9 @@ +import path from 'path'; import * as semver from 'semver'; +import * as tar from 'tar'; import { request } from '../spec-utils/httpRequest'; import { Log, LogLevel } from '../spec-utils/log'; +import { mkdirpLocal, writeLocalFile } from '../spec-utils/pfs'; export const DEVCONTAINER_MANIFEST_MEDIATYPE = 'application/vnd.devcontainers'; export const DEVCONTAINER_TAR_LAYER_MEDIATYPE = 'application/vnd.devcontainers.layer.v1+tar'; @@ -265,3 +268,61 @@ export async function getPublishedVersions(ref: OCIRef, output: Log, sorted: boo return undefined; } } + +export async function getBlob(output: Log, env: NodeJS.ProcessEnv, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, authToken?: string, ignoredFilesDuringExtraction: string[] = []): Promise { + // TODO: Parallelize if multiple layers (not likely). + // TODO: Seeking might be needed if the size is too large. + try { + await mkdirpLocal(ociCacheDir); + const tempTarballPath = path.join(ociCacheDir, 'blob.tar'); + + const headers: HEADERS = { + 'user-agent': 'devcontainer', + 'accept': 'application/vnd.oci.image.manifest.v1+json', + }; + + const auth = authToken ?? await fetchRegistryAuthToken(output, ociRef.registry, ociRef.path, env, 'pull'); + if (auth) { + headers['authorization'] = `Bearer ${auth}`; + } + + const options = { + type: 'GET', + url: url, + headers: headers + }; + + const blob = await request(options, output); + + await mkdirpLocal(destCachePath); + await writeLocalFile(tempTarballPath, blob); + + const files: string[] = []; + await tar.x( + { + file: tempTarballPath, + cwd: destCachePath, + filter: (path: string, stat: tar.FileStat) => { + // Skip files that are in the ignore list + if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) { + // Skip. + output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace); + return false; + } + // Keep track of all files extracted, in case the caller is interested. + output.write(`${path} : ${stat.type}`, LogLevel.Trace); + if ((stat.type.toString() === 'File')) { + files.push(path); + } + return true; + } + } + ); + + output.write('Files extracted from blob: ' + files.join(', '), LogLevel.Trace); + return files; + } catch (e) { + output.write(`error: ${e}`, LogLevel.Error); + return undefined; + } +} diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index fbe70f383..ba439186a 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -1,10 +1,6 @@ -import * as path from 'path'; -import * as tar from 'tar'; -import { request } from '../spec-utils/httpRequest'; import { Log, LogLevel } from '../spec-utils/log'; -import { mkdirpLocal, writeLocalFile } from '../spec-utils/pfs'; import { Feature, FeatureSet } from './containerFeaturesConfiguration'; -import { fetchOCIManifestIfExists, fetchRegistryAuthToken, getRef, HEADERS, OCIManifest, OCIRef } from './containerCollectionsOCI'; +import { fetchOCIManifestIfExists, getBlob, getRef, OCIManifest } from './containerCollectionsOCI'; export function getOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: OCIManifest, originalUserFeatureId: string): FeatureSet { @@ -51,52 +47,11 @@ export async function fetchOCIFeature(output: Log, env: NodeJS.ProcessEnv, featu const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`; output.write(`blob url: ${blobUrl}`, LogLevel.Trace); - const success = await getFeatureBlob(output, env, blobUrl, ociCacheDir, featCachePath, featureRef); + const files = await getBlob(output, env, blobUrl, ociCacheDir, featCachePath, featureRef); - if (!success) { + if (!files) { throw new Error(`Failed to download package for ${featureSet.sourceInformation.featureRef.resource}`); } return true; } - -// Downloads a blob from a registry. -export async function getFeatureBlob(output: Log, env: NodeJS.ProcessEnv, url: string, ociCacheDir: string, featCachePath: string, featureRef: OCIRef, authToken?: string): Promise { - // TODO: Parallelize if multiple layers (not likely). - // TODO: Seeking might be needed if the size is too large. - try { - const tempTarballPath = path.join(ociCacheDir, 'blob.tar'); - - const headers: HEADERS = { - 'user-agent': 'devcontainer', - 'accept': 'application/vnd.oci.image.manifest.v1+json', - }; - - const auth = authToken ?? await fetchRegistryAuthToken(output, featureRef.registry, featureRef.path, env, 'pull'); - if (auth) { - headers['authorization'] = `Bearer ${auth}`; - } - - const options = { - type: 'GET', - url: url, - headers: headers - }; - - const blob = await request(options, output); - - await mkdirpLocal(featCachePath); - await writeLocalFile(tempTarballPath, blob); - await tar.x( - { - file: tempTarballPath, - cwd: featCachePath, - } - ); - - return true; - } catch (e) { - output.write(`error: ${e}`, LogLevel.Error); - return false; - } -} diff --git a/src/spec-configuration/containerTemplatesOCI.ts b/src/spec-configuration/containerTemplatesOCI.ts new file mode 100644 index 000000000..91a96d58b --- /dev/null +++ b/src/spec-configuration/containerTemplatesOCI.ts @@ -0,0 +1,68 @@ +import { Log, LogLevel } from '../spec-utils/log'; +import * as os from 'os'; +import * as path from 'path'; +import { fetchOCIManifestIfExists, getBlob, getRef, OCIManifest } from './containerCollectionsOCI'; +import { isLocalFile, readLocalFile, writeLocalFile } from '../spec-utils/pfs'; + +export interface TemplateOptions { + [name: string]: string; +} + +export interface SelectedTemplate { + id: string; + options: TemplateOptions; +} + +export async function fetchTemplate(output: Log, selectedTemplate: SelectedTemplate, templateDestPath: string): Promise { + const { id, options } = selectedTemplate; + const templateRef = getRef(output, id); + if (!templateRef) { + output.write(`Failed to parse template ref for ${id}`, LogLevel.Error); + return; + } + + const ociManifest = await fetchOCITemplateManifestIfExistsFromUserIdentifier(output, process.env, id); + if (!ociManifest) { + output.write(`Failed to fetch template manifest for ${id}`, LogLevel.Error); + return; + } + + const blobUrl = `https://${templateRef.registry}/v2/${templateRef.path}/blobs/${ociManifest?.layers[0].digest}`; + output.write(`blob url: ${blobUrl}`, LogLevel.Trace); + + const tmpDir = path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`); + const files = await getBlob(output, process.env, blobUrl, tmpDir, templateDestPath, templateRef, undefined, ['devcontainer-template.json', 'README.md', 'NOTES.md']); + + if (!files) { + throw new Error(`Failed to download package for ${templateRef.resource}`); + } + + // Scan all template files and replace any templated values. + for (const f of files) { + output.write(`Scanning file '${f}'`, LogLevel.Trace); + const filePath = path.join(templateDestPath, f); + if (await isLocalFile(filePath)) { + const fileContents = await readLocalFile(filePath); + const fileContentsReplaced = replaceTemplatedValues(output, fileContents.toString(), options); + await writeLocalFile(filePath, Buffer.from(fileContentsReplaced)); + } else { + output.write(`Could not find templated file '${f}'.`, LogLevel.Error); + } + } + + return files; +} + + +async function fetchOCITemplateManifestIfExistsFromUserIdentifier(output: Log, env: NodeJS.ProcessEnv, identifier: string, manifestDigest?: string, authToken?: string): Promise { + const templateRef = getRef(output, identifier); + return await fetchOCIManifestIfExists(output, env, templateRef, manifestDigest, authToken); +} + +function replaceTemplatedValues(output: Log, template: string, options: TemplateOptions) { + const pattern = /\${templateOption:\s*(\w+?)\s*}/g; // ${templateOption:XXXX} + return template.replace(pattern, (_, token) => { + output.write(`Replacing ${token} with ${options[token]}`); + return options[token] || ''; + }); +} \ No newline at end of file diff --git a/src/test/container-features/containerFeaturesOCI.test.ts b/src/test/container-features/containerFeaturesOCI.test.ts index a8c46be45..ad7820320 100644 --- a/src/test/container-features/containerFeaturesOCI.test.ts +++ b/src/test/container-features/containerFeaturesOCI.test.ts @@ -1,6 +1,5 @@ import { assert } from 'chai'; -import { getRef, getManifest } from '../../spec-configuration/containerCollectionsOCI'; -import { getFeatureBlob } from '../../spec-configuration/containerFeaturesOCI'; +import { getRef, getManifest, getBlob } from '../../spec-configuration/containerCollectionsOCI'; import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); @@ -43,7 +42,7 @@ describe('Test OCI Pull', () => { it('Download a feature', async () => { const featureRef = getRef(output, 'ghcr.io/codspace/features/ruby:1.0.13'); - const result = await getFeatureBlob(output, process.env, 'https://ghcr.io/v2/codspace/features/ruby/blobs/sha256:8f59630bd1ba6d9e78b485233a0280530b3d0a44338f472206090412ffbd3efb', '/tmp', '/tmp/featureTest', featureRef); - assert.isTrue(result); + const files = await getBlob(output, process.env, 'https://ghcr.io/v2/codspace/features/ruby/blobs/sha256:8f59630bd1ba6d9e78b485233a0280530b3d0a44338f472206090412ffbd3efb', '/tmp', '/tmp/featureTest', featureRef); + assert.isArray(files); }); }); \ No newline at end of file diff --git a/src/test/container-templates/containerTemplatesOCI.test.ts b/src/test/container-templates/containerTemplatesOCI.test.ts new file mode 100644 index 000000000..2ac92deb9 --- /dev/null +++ b/src/test/container-templates/containerTemplatesOCI.test.ts @@ -0,0 +1,55 @@ +import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; +import * as assert from 'assert'; +export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); +import { fetchTemplate, SelectedTemplate } from '../../spec-configuration/containerTemplatesOCI'; +import * as path from 'path'; +import { readLocalFile } from '../../spec-utils/pfs'; + +describe('fetchTemplate', async function () { + this.timeout('120s'); + + it('succeeds on docker-from-docker template', async () => { + + // https://github.com/devcontainers/templates/tree/main/src/docker-from-docker + const selectedTemplate: SelectedTemplate = { + id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest', + options: {'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true', 'enableNonRootDocker': 'true' } + }; + + const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp1')); + const files = await fetchTemplate(output, selectedTemplate, dest); + assert.ok(files); + // Should only container 1 file '.devcontainer.json'. The other 3 in this repo should be ignored. + assert.strictEqual(files.length, 1); + + // Read File + const file = (await readLocalFile(path.join(dest, files[0]))).toString(); + assert.match(file, /"name": "Docker from Docker"/); + assert.match(file, /"installZsh": "false"/); + assert.match(file, /"upgradePackages": "true"/); + assert.match(file, /"version": "20.10"/); + assert.match(file, /"moby": "true"/); + assert.match(file, /"enableNonRootDocker": "true"/); + }); + + it('succeeds on anaconda-postgres template', async () => { + + // https://github.com/devcontainers/templates/tree/main/src/anaconda-postgres + const selectedTemplate: SelectedTemplate = { + id: 'ghcr.io/devcontainers/templates/anaconda-postgres:latest', + options: { 'nodeVersion': 'lts/*' } + }; + + const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp2')); + const files = await fetchTemplate(output, selectedTemplate, dest); + assert.ok(files); + // Expected: + // ./environment.yml, ./.devcontainer/.env, ./.devcontainer/Dockerfile, ./.devcontainer/devcontainer.json, ./.devcontainer/docker-compose.yml, ./.devcontainer/noop.txt + assert.strictEqual(files.length, 6); + + // Read File + const file = (await readLocalFile(path.join(dest, '.devcontainer', 'Dockerfile'))).toString(); + assert.match(file, /ARG NODE_VERSION="lts\/\*"/); + + }); +}); \ No newline at end of file