From 757b885a79502bddff04dbd2de2370666b8f8e57 Mon Sep 17 00:00:00 2001 From: Matthew Mulholland Date: Sun, 3 Jun 2018 10:35:00 +1000 Subject: [PATCH] #804: corrected path for all OS. updated zip libraries for better stream error handling. Corrected logic for multiple ignores. --- package.json | 4 +- src/renderer/frictionlessDataPackage.js | 2 +- src/renderer/importPackage.js | 77 +++++++++++++++---------- yarn.lock | 22 +++---- 4 files changed, 59 insertions(+), 46 deletions(-) diff --git a/package.json b/package.json index b106950fb..25797e262 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "datapackage": "^1.0.6", "electron-settings": "^3.2.0", "escape-regexp": "^0.0.1", - "etl": "^0.5.8", + "etl": "^0.5.12", "fs-extra": "~5.0.0", "handsontable": "^0.35.1", "imports-loader": "^0.8.0", @@ -101,7 +101,7 @@ "svgo": "^1.0.5", "tableschema": "^1.9.1", "tmp": "^0.0.33", - "unzipper": "^0.8.12", + "unzipper": "^0.8.14", "vee-validate": "^2.0.4", "vue": "^2.5.2", "vue-async-computed": "^3.3.1", diff --git a/src/renderer/frictionlessDataPackage.js b/src/renderer/frictionlessDataPackage.js index d708598ce..64897df72 100644 --- a/src/renderer/frictionlessDataPackage.js +++ b/src/renderer/frictionlessDataPackage.js @@ -251,5 +251,5 @@ function addPath(descriptor, tabId) { let parent = 'data' let filename = tabStore.state.tabObjects[tabId].filename let basename = path.basename(filename) - descriptor.path = `${parent}/${basename}` + descriptor.path = path.join(parent, basename) } diff --git a/src/renderer/importPackage.js b/src/renderer/importPackage.js index 4f8212ff4..1d736f966 100644 --- a/src/renderer/importPackage.js +++ b/src/renderer/importPackage.js @@ -7,11 +7,15 @@ import {ipcRenderer as ipc} from 'electron' import {Resource, Package} from 'datapackage' import {dataResourceToFormat} from '@/file-formats.js' +const _ignores = ['__MACOSX'] + // TODO: clean up isTransient logic export async function unzipFile(zipSource, storeCallback, isTransient) { try { let destination = createUnzipDestination(zipSource) + console.log(`destination is: ${destination}`) await fs.ensureDir(destination) + console.log(`directory created: ${destination}`) let processedProperties = await unzipFileToDir(zipSource, destination, isTransient) storeCallback(processedProperties) } catch (err) { @@ -27,7 +31,8 @@ function createUnzipDestination(zipSource) { async function unzipFileToDir(zipSource, unzipDestination, isTransient) { let processed = {json: [], resource: [], md: []} await fs.createReadStream(zipSource).pipe(unzipper.Parse()).pipe(etl.map(async entry => { - let fileDestination = `${unzipDestination}/${entry.path}` + let fileDestination = path.join(unzipDestination, entry.path) + console.log(`file destination is: ${fileDestination}`) await processStream(entry, processed, fileDestination) })).promise() validateMdFile(processed) @@ -49,36 +54,47 @@ async function getDataPackageJson(processed) { } async function processStream(entry, processed, fileDestination) { - for (const ignore of ['__MACOSX']) { + console.log(`entry path is ${entry.path}`) + console.log(`file destination is: ${fileDestination}`) + if (isIgnored(fileDestination)) { + await cleanUp(entry, fileDestination) + } else { + switch (path.extname(entry.path)) { + case '.csv': + case '.tsv': + await fs.ensureFile(fileDestination) + await unzippedEntryToFile(entry, fileDestination) + processed.resource.push(entry.path) + break + case '.json': + await fs.ensureFile(fileDestination) + await unzippedEntryToFile(entry, fileDestination) + processed.json.push(fileDestination) + processed.parentFolders = path.dirname(entry.path) + break + case '.md': + await fs.ensureFile(fileDestination) + await unzippedEntryToFile(entry, fileDestination) + let textMd = await stringify(fileDestination) + setProvenance(textMd) + processed.md.push(fileDestination) + break + default: + entry.autodrain() + break + } + } +} + +function isIgnored(fileDestination) { + let shouldIgnore = false + for (const ignore of _ignores) { if (fileDestination.includes(ignore)) { - await cleanUp(entry, fileDestination) - } else { - switch (path.extname(entry.path)) { - case '.csv': - case '.tsv': - await fs.ensureFile(fileDestination) - await unzippedEntryToFile(entry, fileDestination) - processed.resource.push(entry.path) - break - case '.json': - await fs.ensureFile(fileDestination) - await unzippedEntryToFile(entry, fileDestination) - processed.json.push(fileDestination) - processed.parentFolders = path.dirname(entry.path) - break - case '.md': - await fs.ensureFile(fileDestination) - await unzippedEntryToFile(entry, fileDestination) - let textMd = await stringify(fileDestination) - setProvenance(textMd) - processed.md.push(fileDestination) - break - default: - entry.autodrain() - break - } + shouldIgnore = true + break } } + return shouldIgnore } async function cleanUp(entry, fileDestination) { @@ -152,7 +168,8 @@ async function getHotIdsFromFilenames(processed, unzipDestination, isTransient = let dataPackageJson = processed.json[0] let csvTabs = {} for (let pathname of processed.resource) { - let fileDestination = `${unzipDestination}/${pathname}` + let fileDestination = path.join(unzipDestination, pathname) + console.log(`next opened tab match to `, fileDestination) let tabId = await getTabIdFromFilename(fileDestination) // every processed csv should have a matching tab if (!tabId) { @@ -160,7 +177,7 @@ async function getHotIdsFromFilenames(processed, unzipDestination, isTransient = } let hotId = _.findKey(store.getters.getHotTabs, {tabId: tabId}) // ensure csv path accounts for parent folders zipped up - let re = new RegExp('^' + processed.parentFolders + '/') + let re = new RegExp('^' + processed.parentFolders + path.sep) let resourcePathname = _.replace(pathname, re, '') csvTabs[`${resourcePathname}`] = hotId if (isTransient) { diff --git a/yarn.lock b/yarn.lock index 9b9721252..f69d78eb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1489,8 +1489,8 @@ better-assert@~1.0.0: callsite "1.0.0" big-integer@^1.6.17: - version "1.6.27" - resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.27.tgz#8e56c6f8b2dd6c4fe8d32102b83d4f25868e4b3a" + version "1.6.30" + resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.30.tgz#7796f04acdd6ba56345f19049c8fffd427f09d16" big.js@^3.1.3: version "3.2.0" @@ -4147,14 +4147,14 @@ etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" -etl@^0.5.8: - version "0.5.8" - resolved "https://registry.yarnpkg.com/etl/-/etl-0.5.8.tgz#684bf6f20a0a23d31b75e3d63f56848dfe511243" +etl@^0.5.12: + version "0.5.12" + resolved "https://registry.yarnpkg.com/etl/-/etl-0.5.12.tgz#71fa98e39706dcd216e5ae09e0f29dfc9f6d128b" dependencies: bluebird "~3.5.0" csv-parser "~1.8.0" duplexer2 "~0.1.4" - moment "~2.18.1" + moment "~2.21.0" streamz "~1.8.4" event-emitter@~0.3.5: @@ -6965,10 +6965,6 @@ moment@2.x, moment@^2.21.0, moment@~2.21.0: version "2.21.0" resolved "https://registry.yarnpkg.com/moment/-/moment-2.21.0.tgz#2a114b51d2a6ec9e6d83cf803f838a878d8a023a" -moment@~2.18.1: - version "2.18.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" - move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -10228,9 +10224,9 @@ unzip-response@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97" -unzipper@^0.8.12: - version "0.8.12" - resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.8.12.tgz#178de4e263d96a2550fb6f4804d26c06edb9c8bd" +unzipper@^0.8.14: + version "0.8.14" + resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.8.14.tgz#ade0524cd2fc14d11b8de258be22f9d247d3f79b" dependencies: big-integer "^1.6.17" binary "~0.3.0"