diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..d186eba --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,19 @@ +name: test + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: 15.x + - run: npm install + - run: npm run build --if-present + - run: npm test diff --git a/README.md b/README.md index 995932d..5d2fe9c 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,13 @@ console.info('go-ipfs is installed at', path()) An error will be thrown if the path to the binary cannot be resolved. +### Caching + +Downloaded archives are placed in OS-specific cache directory which can be customized by setting `NPM_GO_IPFS_CACHE` in env. + ## Development -**Warning**: the file `bin/ipfs` is a placeholder, when downloading stuff, it gets replaced. so if you run `node install.js` it will then be dirty in the git repo. **Do not commit this file**, as then you would be commiting a big binary and publishing it to npm. (**TODO: add a pre-commit or pre-publish hook that warns about this**) +**Warning**: the file `bin/ipfs` is a placeholder, when downloading stuff, it gets replaced. so if you run `node install.js` it will then be dirty in the git repo. **Do not commit this file**, as then you would be commiting a big binary and publishing it to npm. A pre-commit hook exists and should protect against this, but better safe than sorry. ### Publish a new version diff --git a/package.json b/package.json index 754b033..44c0bcf 100644 --- a/package.json +++ b/package.json @@ -5,9 +5,11 @@ "main": "src/index.js", "scripts": { "postinstall": "node src/post-install.js", + "restore-bin": "git restore --source=HEAD --staged --worktree -- bin/ipfs", "test": "tape test/*.js | tap-spec", "lint": "standard" }, + "pre-commit": "restore-bin", "bin": { "ipfs": "bin/ipfs" }, @@ -28,15 +30,18 @@ "devDependencies": { "execa": "^4.0.1", "fs-extra": "^9.0.0", + "pre-commit": "^1.2.2", "standard": "^13.1.0", "tap-spec": "^5.0.0", "tape": "^4.13.2", "tape-promise": "^4.0.0" }, "dependencies": { + "cachedir": "^2.3.0", "go-platform": "^1.0.0", + "got": "^11.7.0", "gunzip-maybe": "^1.4.2", - "node-fetch": "^2.6.0", + "hasha": "^5.2.2", "pkg-conf": "^3.1.0", "tar-fs": "^2.1.0", "unzip-stream": "^0.3.0" diff --git a/src/download.js b/src/download.js index 7dfad8c..401a766 100644 --- a/src/download.js +++ b/src/download.js @@ -15,16 +15,60 @@ */ const goenv = require('go-platform') const gunzip = require('gunzip-maybe') +const got = require('got') const path = require('path') const tarFS = require('tar-fs') const unzip = require('unzip-stream') -const fetch = require('node-fetch') const pkgConf = require('pkg-conf') +const cachedir = require('cachedir') const pkg = require('../package.json') const fs = require('fs') +const hasha = require('hasha') const cproc = require('child_process') const isWin = process.platform === 'win32' +// avoid expensive fetch if file is already in cache +async function cachingFetchAndVerify (url) { + const cacheDir = process.env.NPM_GO_IPFS_CACHE || cachedir('npm-go-ipfs') + const filename = url.split('/').pop() + const cachedFilePath = path.join(cacheDir, filename) + const cachedHashPath = `${cachedFilePath}.sha512` + + if (!fs.existsSync(cacheDir)) { + fs.mkdirSync(cacheDir, { recursive: true }) + } + if (!fs.existsSync(cachedFilePath)) { + console.info(`Downloading ${url} to ${cacheDir}`) + // download file + fs.writeFileSync(cachedFilePath, await got(url).buffer()) + console.info(`Downloaded ${url}`) + + // ..and checksum + console.info(`Downloading ${filename}.sha512`) + fs.writeFileSync(cachedHashPath, await got(`${url}.sha512`).buffer()) + console.info(`Downloaded ${filename}.sha512`) + } else { + console.info(`Found ${cachedFilePath}`) + } + + console.info(`Verifying ${filename}.sha512`) + + const digest = Buffer.alloc(128) + const fd = fs.openSync(cachedHashPath, 'r') + fs.readSync(fd, digest, 0, digest.length, 0) + fs.closeSync(fd) + const expectedSha = digest.toString('utf8') + const calculatedSha = await hasha.fromFile(cachedFilePath, { encoding: 'hex', algorithm: 'sha512' }) + if (calculatedSha !== expectedSha) { + console.log(`Expected SHA512: ${expectedSha}`) + console.log(`Calculated SHA512: ${calculatedSha}`) + throw new Error(`SHA512 of ${cachedFilePath}' (${calculatedSha}) does not match expected value from ${cachedFilePath}.sha512 (${expectedSha})`) + } + console.log(`OK (${expectedSha})`) + + return fs.createReadStream(cachedFilePath) +} + function unpack (url, installPath, stream) { return new Promise((resolve, reject) => { if (url.endsWith('.zip')) { @@ -66,13 +110,8 @@ function cleanArguments (version, platform, arch, installPath) { } async function ensureVersion (version, distUrl) { - const res = await fetch(`${distUrl}/go-ipfs/versions`) console.info(`${distUrl}/go-ipfs/versions`) - if (!res.ok) { - throw new Error(`Unexpected status: ${res.status}`) - } - - const versions = (await res.text()).trim().split('\n') + const versions = (await got(`${distUrl}/go-ipfs/versions`).text()).trim().split('\n') if (versions.indexOf(version) === -1) { throw new Error(`Version '${version}' not available`) @@ -82,9 +121,7 @@ async function ensureVersion (version, distUrl) { async function getDownloadURL (version, platform, arch, distUrl) { await ensureVersion(version, distUrl) - const res = await fetch(`${distUrl}/go-ipfs/${version}/dist.json`) - if (!res.ok) throw new Error(`Unexpected status: ${res.status}`) - const data = await res.json() + const data = await got(`${distUrl}/go-ipfs/${version}/dist.json`).json() if (!data.platforms[platform]) { throw new Error(`No binary available for platform '${platform}'`) @@ -100,19 +137,9 @@ async function getDownloadURL (version, platform, arch, distUrl) { async function download ({ version, platform, arch, installPath, distUrl }) { const url = await getDownloadURL(version, platform, arch, distUrl) + const data = await cachingFetchAndVerify(url) - console.info(`Downloading ${url}`) - - const res = await fetch(url) - - if (!res.ok) { - throw new Error(`Unexpected status: ${res.status}`) - } - - console.info(`Downloaded ${url}`) - - await unpack(url, installPath, res.body) - + await unpack(url, installPath, data) console.info(`Unpacked ${installPath}`) return path.join(installPath, 'go-ipfs', `ipfs${platform === 'windows' ? '.exe' : ''}`) diff --git a/test/fixtures/example-project/package.json b/test/fixtures/example-project/package.json index ba3d42c..15b08c8 100644 --- a/test/fixtures/example-project/package.json +++ b/test/fixtures/example-project/package.json @@ -7,8 +7,5 @@ "license": "ISC", "dependencies": { "go-ipfs": "file://../../../" - }, - "go-ipfs": { - "version": "v0.4.20" } } diff --git a/test/install.js b/test/install.js index 6f7c2cd..44c61ca 100644 --- a/test/install.js +++ b/test/install.js @@ -2,31 +2,43 @@ const fs = require('fs-extra') const path = require('path') const test = require('tape') const execa = require('execa') +const cachedir = require('cachedir') /* - Test that go-ipfs is downloaded during npm install. - - package up the current source code with `npm pack` - - install the tarball into the example project - - ensure that the "go-ipfs.version" prop in the package.json is used + Test that correct go-ipfs is downloaded during npm install. */ -const testVersion = require('./fixtures/example-project/package.json')['go-ipfs'].version +const expectedVersion = require('../package.json').version async function clean () { await fs.remove(path.join(__dirname, 'fixtures', 'example-project', 'node_modules')) await fs.remove(path.join(__dirname, 'fixtures', 'example-project', 'package-lock.json')) + await fs.remove(cachedir('npm-go-ipfs')) } test.onFinish(clean) -test('Ensure go-ipfs.version defined in parent package.json is used', async (t) => { +test('Ensure go-ipfs defined in package.json is fetched on dependency install', async (t) => { await clean() + const exampleProjectRoot = path.join(__dirname, 'fixtures', 'example-project') + // from `example-project`, install the module const res = execa.sync('npm', ['install'], { - cwd: path.join(__dirname, 'fixtures', 'example-project') + cwd: exampleProjectRoot + }) + + // confirm package.json is correct + const fetchedVersion = require(path.join(exampleProjectRoot, 'node_modules', 'go-ipfs', 'package.json')).version + t.ok(expectedVersion === fetchedVersion, `package.json versions match '${expectedVersion}'`) + + // confirm binary is correct + const binary = path.join(exampleProjectRoot, 'node_modules', 'go-ipfs', 'bin', 'ipfs') + const versionRes = execa.sync(binary, ['--version'], { + cwd: exampleProjectRoot }) - const msg = `Downloading https://dist.ipfs.io/go-ipfs/${testVersion}` - t.ok(res.stdout.includes(msg), msg) + + t.ok(versionRes.stdout === `ipfs version ${expectedVersion}`, `ipfs --version output match '${expectedVersion}'`) + t.end() })