#804: corrected path for all OS. updated zip libraries for better str…

…eam error handling. Corrected logic for multiple ignores.

package.json

@@ -84,7 +84,7 @@
     "datapackage": "^1.0.6",
     "electron-settings": "^3.2.0",
     "escape-regexp": "^0.0.1",
-    "etl": "^0.5.8",
+    "etl": "^0.5.12",
     "fs-extra": "~5.0.0",
     "handsontable": "^0.35.1",
     "imports-loader": "^0.8.0",
@@ -101,7 +101,7 @@
     "svgo": "^1.0.5",
     "tableschema": "^1.9.1",
     "tmp": "^0.0.33",
-    "unzipper": "^0.8.12",
+    "unzipper": "^0.8.14",
     "vee-validate": "^2.0.4",
     "vue": "^2.5.2",
     "vue-async-computed": "^3.3.1",

src/renderer/frictionlessDataPackage.js

@@ -251,5 +251,5 @@ function addPath(descriptor, tabId) {
   let parent = 'data'
   let filename = tabStore.state.tabObjects[tabId].filename
   let basename = path.basename(filename)
-  descriptor.path = `${parent}/${basename}`
+  descriptor.path = path.join(parent, basename)
 }

src/renderer/importPackage.js

@@ -7,11 +7,15 @@ import {ipcRenderer as ipc} from 'electron'
 import {Resource, Package} from 'datapackage'
 import {dataResourceToFormat} from '@/file-formats.js'
 
+const _ignores = ['__MACOSX']
+
 // TODO: clean up isTransient logic
 export async function unzipFile(zipSource, storeCallback, isTransient) {
   try {
     let destination = createUnzipDestination(zipSource)
+    console.log(`destination is: ${destination}`)
     await fs.ensureDir(destination)
+    console.log(`directory created: ${destination}`)
     let processedProperties = await unzipFileToDir(zipSource, destination, isTransient)
     storeCallback(processedProperties)
   } catch (err) {
@@ -27,7 +31,8 @@ function createUnzipDestination(zipSource) {
 async function unzipFileToDir(zipSource, unzipDestination, isTransient) {
   let processed = {json: [], resource: [], md: []}
   await fs.createReadStream(zipSource).pipe(unzipper.Parse()).pipe(etl.map(async entry => {
-    let fileDestination = `${unzipDestination}/${entry.path}`
+    let fileDestination = path.join(unzipDestination, entry.path)
+    console.log(`file destination is: ${fileDestination}`)
     await processStream(entry, processed, fileDestination)
   })).promise()
   validateMdFile(processed)
@@ -49,36 +54,47 @@ async function getDataPackageJson(processed) {
 }
 
 async function processStream(entry, processed, fileDestination) {
-  for (const ignore of ['__MACOSX']) {
+  console.log(`entry path is ${entry.path}`)
+  console.log(`file destination is: ${fileDestination}`)
+  if (isIgnored(fileDestination)) {
+    await cleanUp(entry, fileDestination)
+  } else {
+    switch (path.extname(entry.path)) {
+      case '.csv':
+      case '.tsv':
+        await fs.ensureFile(fileDestination)
+        await unzippedEntryToFile(entry, fileDestination)
+        processed.resource.push(entry.path)
+        break
+      case '.json':
+        await fs.ensureFile(fileDestination)
+        await unzippedEntryToFile(entry, fileDestination)
+        processed.json.push(fileDestination)
+        processed.parentFolders = path.dirname(entry.path)
+        break
+      case '.md':
+        await fs.ensureFile(fileDestination)
+        await unzippedEntryToFile(entry, fileDestination)
+        let textMd = await stringify(fileDestination)
+        setProvenance(textMd)
+        processed.md.push(fileDestination)
+        break
+      default:
+        entry.autodrain()
+        break
+    }
+  }
+}
+
+function isIgnored(fileDestination) {
+  let shouldIgnore = false
+  for (const ignore of _ignores) {
     if (fileDestination.includes(ignore)) {
-      await cleanUp(entry, fileDestination)
-    } else {
-      switch (path.extname(entry.path)) {
-        case '.csv':
-        case '.tsv':
-          await fs.ensureFile(fileDestination)
-          await unzippedEntryToFile(entry, fileDestination)
-          processed.resource.push(entry.path)
-          break
-        case '.json':
-          await fs.ensureFile(fileDestination)
-          await unzippedEntryToFile(entry, fileDestination)
-          processed.json.push(fileDestination)
-          processed.parentFolders = path.dirname(entry.path)
-          break
-        case '.md':
-          await fs.ensureFile(fileDestination)
-          await unzippedEntryToFile(entry, fileDestination)
-          let textMd = await stringify(fileDestination)
-          setProvenance(textMd)
-          processed.md.push(fileDestination)
-          break
-        default:
-          entry.autodrain()
-          break
-      }
+      shouldIgnore = true
+      break
     }
   }
+  return shouldIgnore
 }
 
 async function cleanUp(entry, fileDestination) {
@@ -152,15 +168,16 @@ async function getHotIdsFromFilenames(processed, unzipDestination, isTransient =
   let dataPackageJson = processed.json[0]
   let csvTabs = {}
   for (let pathname of processed.resource) {
-    let fileDestination = `${unzipDestination}/${pathname}`
+    let fileDestination = path.join(unzipDestination, pathname)
+    console.log(`next opened tab match to `, fileDestination)
     let tabId = await getTabIdFromFilename(fileDestination)
     // every processed csv should have a matching tab
     if (!tabId) {
       throw new Error(`There was a problem matching ${fileDestination} with an opened tab.`)
     }
     let hotId = _.findKey(store.getters.getHotTabs, {tabId: tabId})
     // ensure csv path accounts for parent folders zipped up
-    let re = new RegExp('^' + processed.parentFolders + '/')
+    let re = new RegExp('^' + processed.parentFolders + path.sep)
     let resourcePathname = _.replace(pathname, re, '')
     csvTabs[`${resourcePathname}`] = hotId
     if (isTransient) {

yarn.lock

@@ -1489,8 +1489,8 @@ better-assert@~1.0.0:
     callsite "1.0.0"
 
 big-integer@^1.6.17:
-  version "1.6.27"
-  resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.27.tgz#8e56c6f8b2dd6c4fe8d32102b83d4f25868e4b3a"
+  version "1.6.30"
+  resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.30.tgz#7796f04acdd6ba56345f19049c8fffd427f09d16"
 
 big.js@^3.1.3:
   version "3.2.0"
@@ -4147,14 +4147,14 @@ etag@~1.8.1:
   version "1.8.1"
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
 
-etl@^0.5.8:
-  version "0.5.8"
-  resolved "https://registry.yarnpkg.com/etl/-/etl-0.5.8.tgz#684bf6f20a0a23d31b75e3d63f56848dfe511243"
+etl@^0.5.12:
+  version "0.5.12"
+  resolved "https://registry.yarnpkg.com/etl/-/etl-0.5.12.tgz#71fa98e39706dcd216e5ae09e0f29dfc9f6d128b"
   dependencies:
     bluebird "~3.5.0"
     csv-parser "~1.8.0"
     duplexer2 "~0.1.4"
-    moment "~2.18.1"
+    moment "~2.21.0"
     streamz "~1.8.4"
 
 event-emitter@~0.3.5:
@@ -6965,10 +6965,6 @@ [email protected], moment@^2.21.0, moment@~2.21.0:
   version "2.21.0"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.21.0.tgz#2a114b51d2a6ec9e6d83cf803f838a878d8a023a"
 
-moment@~2.18.1:
-  version "2.18.1"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
-
 move-concurrently@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"
@@ -10228,9 +10224,9 @@ unzip-response@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97"
 
-unzipper@^0.8.12:
-  version "0.8.12"
-  resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.8.12.tgz#178de4e263d96a2550fb6f4804d26c06edb9c8bd"
+unzipper@^0.8.14:
+  version "0.8.14"
+  resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.8.14.tgz#ade0524cd2fc14d11b8de258be22f9d247d3f79b"
   dependencies:
     big-integer "^1.6.17"
     binary "~0.3.0"