Added zip support (#97)

Added zip support

README.md

@@ -162,7 +162,7 @@ It was onle basic introduction to the `Package` class. To learn more let's take
 
 Factory method to instantiate `Package` class. This method is async and it should be used with await keyword or as a `Promise`.
 
-- `descriptor (String/Object)` - data package descriptor as local path, url or object
+- `descriptor (String/Object)` - data package descriptor as local path, url or object. If ththe path has a `zip` file extension it will be unzipped to the temp directory first.
 - `basePath (String)` - base path for all relative paths
 - `strict (Boolean)` - strict flag to alter validation behavior. Setting it to `true` leads to throwing errors on any operation with invalid descriptor
 - `(errors.DataPackageError)` - raises error if something goes wrong
@@ -245,9 +245,7 @@ dataPackage.name // renamed-package
 
 #### `async package.save(target)`
 
-> For now only descriptor will be saved.
-
-Save data package to target destination.
+Save data package to target destination. If target path has a  zip file extension the package will be zipped and saved entirely. If it has a json file extension only the descriptor will be saved.
 
 - `target (String)` - path where to save a data package
 - `(errors.DataPackageError)` - raises error if something goes wrong

data/dp3-zip/data/countries.csv

@@ -0,0 +1,4 @@
+name,size
+gb,100
+us,200
+cn,300

data/dp3-zip/datapackage.json

@@ -0,0 +1,22 @@
+{
+  "name": "abc",
+  "resources": [
+    {
+      "name": "countries",
+      "format": "csv",
+      "path": "data/countries.csv",
+      "schema": {
+        "fields": [
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "size",
+            "type": "number"
+          }
+        ]
+      }
+    }
+  ]
+}

package.json

@@ -38,10 +38,12 @@
     "glob": "^7.1.2",
     "jschardet": "^1.5.1",
     "json-pointer": "^0.6.0",
+    "jszip": "^3.1.5",
     "lodash": "^4.13.1",
     "regenerator-runtime": "^0.11.0",
     "stream-to-async-iterator": "^0.2.0",
     "tableschema": "^1.5.1",
+    "tmp": "0.0.33",
     "tv4": "^1.2.7",
     "url-join": "^2.0.1"
   },

src/package.js

@@ -1,8 +1,11 @@
 const fs = require('fs')
+const JSZip = require('jszip')
 const isEqual = require('lodash/isEqual')
+const isString = require('lodash/isString')
 const isBoolean = require('lodash/isBoolean')
 const cloneDeep = require('lodash/cloneDeep')
 const isUndefined = require('lodash/isUndefined')
+const {promisify} = require('util')
 const {Profile} = require('./profile')
 const {Resource} = require('./resource')
 const {DataPackageError} = require('./errors')
@@ -21,6 +24,14 @@ class Package {
    */
   static async load(descriptor={}, {basePath, strict=false}={}) {
 
+    // Extract zip
+    // TODO:
+    // it's first iteration of the zip loading implementation
+    // for now browser support and tempdir cleanup (not needed?) is not covered
+    if (isString(descriptor) && descriptor.endsWith('.zip')) {
+      descriptor = await extractZip(descriptor)
+    }
+
     // Get base path
     if (isUndefined(basePath)) {
       basePath = helpers.locateDescriptor(descriptor)
@@ -177,8 +188,43 @@ class Package {
    */
   save(target) {
     return new Promise((resolve, reject) => {
-      const contents = JSON.stringify(this._currentDescriptor, null, 4)
-      fs.writeFile(target, contents, error => (!error) ? resolve() : reject(error))
+
+      // Save descriptor to json
+      if (target.endsWith('.json')) {
+        const contents = JSON.stringify(this._currentDescriptor, null, 4)
+        fs.writeFile(target, contents, error => (!error) ? resolve() : reject(error))
+
+      // Save package to zip
+      } else {
+
+        // Not supported in browser
+        if (config.IS_BROWSER) {
+          throw new DataPackageError('Zip is not supported in browser')
+        }
+
+        // Prepare zip
+        const zip = new JSZip()
+        const descriptor = cloneDeep(this._currentDescriptor)
+        for (const [index, resource] of this.resources.entries()) {
+          if (!resource.name) continue
+          if (!resource.local) continue
+          let path = `data/${resource.name}`
+          const format = resource.descriptor.format
+          if (format) path = `${path}.${format.toLowerCase()}`
+          descriptor.resources[index].path = path
+          zip.file(path, resource.rawRead())
+        }
+        zip.file('datapackage.json', JSON.stringify(descriptor, null, 4))
+
+        // Write zip
+        zip
+          .generateNodeStream({type: 'nodebuffer', streamFiles: true})
+          .pipe(fs.createWriteStream(target).on('error', error => reject(error)))
+          .on('error', error => reject(error))
+          .on('finish', () => resolve(true))
+
+      }
+
     })
   }
 
@@ -247,6 +293,49 @@ class Package {
 
 // Internal
 
+async function extractZip(descriptor) {
+
+  // Not supported in browser
+  if (config.IS_BROWSER) {
+    throw new DataPackageError('Zip is not supported in browser')
+  }
+
+  // Load zip
+  const zip = JSZip()
+  const tempdir = await promisify(require('tmp').dir)()
+  await zip.loadAsync(promisify(fs.readFile)(descriptor))
+
+  // Validate zip
+  if (!zip.files['datapackage.json']) {
+    throw new DataPackageError('Invalid zip with data package')
+  }
+
+  // Save zip to tempdir
+  for (const [name, item] of Object.entries(zip.files)) {
+
+    // Get path/descriptor
+    const path = `${tempdir}/${name}`
+    if (path.endsWith('datapackage.json')) {
+      descriptor = path
+    }
+
+    // Directory
+    if (item.dir) {
+      await promisify(fs.mkdir)(path)
+
+    // File
+    } else {
+      const contents = await item.async('nodebuffer')
+      await promisify(fs.writeFile)(path, contents)
+    }
+
+  }
+
+  return descriptor
+
+}
+
+
 function findFiles(pattern, basePath) {
   const glob = require('glob')
   return new Promise((resolve, reject) => {

test/package.js

@@ -1,7 +1,9 @@
 const fs = require('fs')
+const JSZip = require('jszip')
 const axios = require('axios')
 const sinon = require('sinon')
 const {assert} = require('chai')
+const {promisify} = require('util')
 const {catchError} = require('./helpers')
 const cloneDeep = require('lodash/cloneDeep')
 const AxiosMock = require('axios-mock-adapter')
@@ -674,4 +676,51 @@ describe('Package', () => {
 
   })
 
+  describe('#zip', () => {
+
+    it('should load package from a zip', async function() {
+      if (process.env.USER_ENV === 'browser') this.skip()
+      const dp = await Package.load('data/dp3-zip.zip')
+      const countries = await dp.getResource('countries').read({keyed: true})
+      assert.deepEqual(dp.descriptor.name, 'abc')
+      assert.deepEqual(countries, [
+        {name: 'gb', size: 100},
+        {name: 'us', size: 200},
+        {name: 'cn', size: 300},
+      ])
+    })
+
+    it('should save package as a zip', async function() {
+      if (process.env.USER_ENV === 'browser') this.skip()
+
+      // Save as a zip
+      const dp = await Package.load('data/dp3-zip/datapackage.json')
+      const target = await promisify(require('tmp').file)({postfix: '.zip'})
+      const result = await dp.save(target)
+      assert.ok(result)
+
+      // Assert file names
+      const zip = JSZip()
+      await zip.loadAsync(promisify(fs.readFile)(target))
+      assert.deepEqual(zip.file('datapackage.json').name, 'datapackage.json')
+      assert.deepEqual(zip.file('data/countries.csv').name, 'data/countries.csv')
+
+      // Assert contents
+      const descContents = await zip.file('datapackage.json').async('string')
+      const dataContents = await zip.file('data/countries.csv').async('string')
+      assert.deepEqual(JSON.parse(descContents), dp.descriptor)
+      assert.deepEqual(dataContents, 'name,size\ngb,100\nus,200\ncn,300\n')
+
+    })
+
+    it('should raise saving package as a zip to the bad path', async function() {
+      if (process.env.USER_ENV === 'browser') this.skip()
+      const dp = await Package.load('data/dp3-zip/datapackage.json')
+      const error = await catchError(dp.save.bind(dp), 'non-existent/datapackage.zip')
+      assert.include(error.message, 'no such file or directory')
+      assert.include(error.message, 'non-existent/datapackage.zip')
+    })
+
+  })
+
 })

webpack.config.js

@@ -31,6 +31,7 @@ let webpackConfig = {
     fs: 'empty',
     http: 'empty',
     https: 'empty',
+    crypto: 'empty',
   }
 }