diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index c64f8cfb1..4050e4c0c 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -23,4 +23,4 @@ > Include details about the environment you experienced the problem - this will help us fix the bug quicker. * Data Curator version: 0.Y.Z -* Operating System and version: e.g. macOS High Sierra 10.13.2 +* Operating System and version: e.g. macOS High Sierra 10.13.3 diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 52fe0bf67..af83a0d47 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -16,4 +16,4 @@ ### Your Environment * Data Curator version: 0.Y.Z -* Operating System and version: macOS High Sierra 10.13.2, Windows 7 64bit +* Operating System and version: macOS High Sierra 10.13.3, Windows 7 64bit diff --git a/package.json b/package.json index ddf264841..25de83143 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "DataCurator", - "version": "0.9.0", + "version": "0.10.0", "author": " ", "description": "Data Curator is a simple desktop CSV editor to help describe, validate and share usable open data", "license": "MIT", @@ -82,6 +82,7 @@ "escape-regexp": "^0.0.1", "etl": "^0.5.8", "exports-loader": "^0.6.4", + "fs-extra": "^5.0.0", "handsontable": "^0.34.5", "imports-loader": "^0.7.1", "jquery": "^3.2.1", @@ -93,7 +94,6 @@ "lodash": "^4.17.4", "markdown-it": "^8.4.0", "moment": "^2.19.1", - "node-fs-extra": "^0.8.2", "pikaday": "^1.6.1", "pouchdb-adapter-idb": "^6.3.4", "request": "^2.81.0", @@ -101,7 +101,7 @@ "slug": "^0.9.1", "sortablejs": "^1.6.0", "svgo": "^1.0.0", - "tableschema": "^1.5.1", + "tableschema": "^1.7.0", "temp": "^0.8.3", "unzipper": "^0.8.11", "vee-validate": "^2.0.3", @@ -138,7 +138,7 @@ "devtron": "^1.1.0", "electron": "^1.7.11", "electron-builder": "^19.27.2", - "electron-debug": "^1.5.0", + "electron-debug": "~1.4.0", "electron-devtools-installer": "^2.0.1", "eslint": "^4.10.0", "eslint-config-standard": "^10.2.1", diff --git a/src/main/file.js b/src/main/file.js index 23f4f9720..9107e8e7a 100644 --- a/src/main/file.js +++ b/src/main/file.js @@ -8,12 +8,12 @@ function makeCustomFormat(separator, delimiter) { return { label: 'Custom', filters: [], - options: { - separator: separator, - delimiter: delimiter + dialect: { + delimiter: delimiter, + quoteChar: quoteChar }, - mime_type: 'text/plain', - default_extension: 'txt' + mediatype: 'text/plain', + format: 'txt' } } @@ -27,7 +27,7 @@ function saveAsCustom() { }) ipc.once('formatSelected', function(event, data) { dialog.close() - let format = makeCustomFormat(data.separator, data.delimiter) + let format = makeCustomFormat(data.delimiter, data.quoteChar) saveFileAs(format, currentWindow) }) dialog.loadURL(`http://localhost:9080/#/customformat`) @@ -120,8 +120,8 @@ function openFile(format) { }) } -ipc.on('openFileIntoTab', (event, arg) => { - readFile(arg) +ipc.on('openFileIntoTab', (event, arg1, arg2) => { + readFile(arg1, arg2) }) function readFile(filename, format) { @@ -131,7 +131,7 @@ function readFile(filename, format) { } Fs.readFile(filename, 'utf-8', function(err, data) { if (err) { - console.log(err.stack) + console.log(err) } else { createWindowTabWithFormattedDataFile(data, format, filename) // enableSave() diff --git a/src/main/menu.js b/src/main/menu.js index b6f1da801..0fb3b34bb 100644 --- a/src/main/menu.js +++ b/src/main/menu.js @@ -99,8 +99,13 @@ const template = [ // enabled: false // } // ] - }, { // Placeholder for non-macOS Settings for future feature + // }, { + // type: 'separator' + // }, { + // label: 'Settings', + // enabled: false + }, { type: 'separator' }, { label: 'Save', @@ -243,9 +248,9 @@ const template = [ submenu: [ { label: 'Header Row', + accelerator: 'Shift+CmdOrCtrl+H', type: 'checkbox', checked: false, - accelerator: 'Shift+CmdOrCtrl+H', click(menuItem) { // revert 'checked' toggle so only controlled by header row event menuItem.checked = !menuItem.checked @@ -270,6 +275,7 @@ const template = [ // type: 'separator' // }, { label: 'Guess Column Properties', + accelerator: 'Shift+CmdOrCtrl+G', click: function() { guessColumnProperties() } @@ -277,21 +283,25 @@ const template = [ type: 'separator' }, { label: 'Set Column Properties', + accelerator: 'Shift+CmdOrCtrl+C', click() { triggerMenuButton('Column') } }, { label: 'Set Table Properties', + accelerator: 'Shift+CmdOrCtrl+T', click() { triggerMenuButton('Table') } }, { label: 'Set Provenance Information', + accelerator: 'Shift+CmdOrCtrl+P', click() { triggerMenuButton('Provenance') } }, { label: 'Set Data Package Properties', + accelerator: 'Shift+CmdOrCtrl+D', click() { triggerMenuButton('Package') } @@ -307,7 +317,7 @@ const template = [ type: 'separator' }, { label: 'Export Data Package...', - accelerator: 'CmdOrCtrl+D', + accelerator: 'Shift+CmdOrCtrl+X', click() { triggerMenuButton('Export') } diff --git a/src/renderer/components/CustomFormat.vue b/src/renderer/components/CustomFormat.vue index 710a47068..95f9f6e13 100644 --- a/src/renderer/components/CustomFormat.vue +++ b/src/renderer/components/CustomFormat.vue @@ -1,29 +1,28 @@ diff --git a/src/renderer/components/KeyboardHelp.vue b/src/renderer/components/KeyboardHelp.vue index a992a0bdf..064bcb84d 100644 --- a/src/renderer/components/KeyboardHelp.vue +++ b/src/renderer/components/KeyboardHelp.vue @@ -404,6 +404,36 @@ + + Header Row - Freeze first row and use values for Column Name + Shift ⇧ Ctrl H + Shift ⇧ Command ⌘ H + + + Guess Column Properties - set Type and Format based on column values + Shift ⇧ Ctrl G + Shift ⇧ Command ⌘ G + + + Set Column Properties + Shift ⇧ Ctrl C + Shift ⇧ Command ⌘ C + + + Set Table Properties + Shift ⇧ Ctrl T + Shift ⇧ Command ⌘ T + + + Set Provenance Information + Shift ⇧ Ctrl P + Shift ⇧ Command ⌘ P + + + Set Data Package Properties + Shift ⇧ Ctrl D + Shift ⇧ Command ⌘ D + Validate Table (uses Column Properties if set) Shift ⇧ Ctrl V @@ -411,8 +441,8 @@ Export Data Package - save all data and properties to a .zip file - Ctrl D - Command ⌘ D + Shift ⇧ Ctrl X + Shift ⇧ Command ⌘ X diff --git a/src/renderer/data-actions.js b/src/renderer/data-actions.js index b0c9970e0..f4730df4d 100644 --- a/src/renderer/data-actions.js +++ b/src/renderer/data-actions.js @@ -1,21 +1,31 @@ -import tabStore from '../renderer/store/modules/tabs.js' +import tabStore from '@/store/modules/tabs.js' import fs from 'fs' import {fixRaggedRows} from '@/ragged-rows.js' import {includeHeadersInData} from '@/frictionlessUtilities.js' import {toggleHeaderNoFeedback} from '@/headerRow.js' +import {pushCsvDialect} from '@/dialect.js' // import parse from 'csv-parse/lib/sync' // import stringify from 'csv-stringify' +// TODO : replace jQuery with node 'csv' library's stringify and transform const $ = global.jQuery = require('jquery/dist/jquery.js') require('jquery-csv/src/jquery.csv.js') +var parse = require('csv-parse/lib/sync') +var stringify = require('csv-stringify/lib/sync') +// { delimiter: ',', lineTerminator, quoteChar, doubleQuote, escapeChar, nullSequence, skipInitialSpace, header, caseSensitiveHeader, csvddfVersion } +const frictionlessToCsvmapper = {delimiter: 'delimiter', lineTerminator: 'rowDelimiter', quoteChar: 'quote', escapeChar: 'escape', skipInitialSpace: 'ltrim'} export function loadDataIntoHot(hot, data, format) { let arrays // if no format specified, default to csv if (typeof format === 'undefined' || !format) { - arrays = $.csv.toArrays(data) + arrays = parse(data) } else { - arrays = $.csv.toArrays(data, format.options) + let csvOptions = dialectToCsvOptions(format.dialect) + // TODO: update to stream + arrays = parse(data, csvOptions) + pushCsvDialect(hot.guid, format) } + fixRaggedRows(arrays) hot.loadData(arrays) hot.render() @@ -48,9 +58,24 @@ export function saveDataToFile(hot, format, filename, callback) { let data // if no format specified, default to csv if (typeof format === 'undefined' || !format) { - data = $.csv.fromArrays(arrays) + // TODO: update to stream + data = stringify(arrays) } else { - data = $.csv.fromArrays(arrays, format.options) + let csvOptions = dialectToCsvOptions(format.dialect) + data = stringify(arrays, csvOptions) + pushCsvDialect(hot.guid, format) } fs.writeFile(filename, data, callback) } + +function dialectToCsvOptions(dialect) { + let csvOptions = {} + if (dialect) { + _.forEach(frictionlessToCsvmapper, function(csvKey, frictionlessKey) { + if (_.has(dialect, frictionlessKey)) { + csvOptions[csvKey] = dialect[frictionlessKey] + } + }) + } + return csvOptions +} diff --git a/src/renderer/dialect.js b/src/renderer/dialect.js new file mode 100644 index 000000000..7ff7cb21e --- /dev/null +++ b/src/renderer/dialect.js @@ -0,0 +1,20 @@ +import store from '@/store' + +export function pushCsvDialect(guid, formatOriginal = {}) { + let format = {} + _.assign(format, formatOriginal) + // TODO : neaten up to merge existing values and then push all up + if (format.dialect) { + _.unset(format.dialect, 'objectMode') + _.merge(format.dialect, {header: true}) + _.forEach(format.dialect, function(value, key) { + store.commit('pushTableProperty', {hotId: guid, key: `dialect.${key}`, value: value}) + }) + } + if (format.mediatype) { + store.commit('pushTableProperty', {hotId: guid, key: 'mediatype', value: format.mediatype}) + } + if (format.format) { + store.commit('pushTableProperty', {hotId: guid, key: 'format', value: format.format}) + } +} diff --git a/src/renderer/exportPackage.js b/src/renderer/exportPackage.js index d3124a80c..4187e15c7 100644 --- a/src/renderer/exportPackage.js +++ b/src/renderer/exportPackage.js @@ -7,7 +7,8 @@ import hotStore from '@/store/modules/hots.js' import {extractNameFromFile} from '@/store/tabStoreUtilities.js' const Dialog = remote.dialog -export function createZipFile(json) { +export function createZipFile(text) { + let json = JSON.stringify(text, null, 4) Dialog.showSaveDialog({ filters: [ { diff --git a/src/renderer/file-formats.js b/src/renderer/file-formats.js index 45c9c69aa..92cdac593 100644 --- a/src/renderer/file-formats.js +++ b/src/renderer/file-formats.js @@ -13,12 +13,12 @@ const fileFormats = { extensions: ['csv'] } ], - options: { - separator: ',', - delimiter: '"' + dialect: { + delimiter: ',', + quoteChar: '"' }, - mime_type: 'text/csv', - default_extension: 'csv' + mediatype: 'text/csv', + format: 'csv' }, tsv: { label: 'Tab separated...', @@ -34,12 +34,12 @@ const fileFormats = { extensions: ['dat'] } ], - options: { - separator: '\t', - delimiter: '"' + dialect: { + delimiter: '\t', + quoteChar: '"' }, - mime_type: 'text/tab-separated-values', - default_extension: 'tsv' + mediatype: 'text/tab-separated-values', + format: 'tsv' }, semicolon: { label: 'Semicolon separated...', @@ -49,12 +49,12 @@ const fileFormats = { extensions: ['csv'] } ], - options: { - separator: ';', - delimiter: '"' + dialect: { + delimiter: ';', + quoteChar: '"' }, - mime_type: 'text/csv', - default_extension: 'csv' + mediatype: 'text/csv', + format: 'csv' } } diff --git a/src/renderer/frictionless.js b/src/renderer/frictionless.js index 569a392d0..0aeb6f35e 100644 --- a/src/renderer/frictionless.js +++ b/src/renderer/frictionless.js @@ -1,20 +1,19 @@ import {Table, Schema} from 'tableschema' import {HotRegister} from '@/hot.js' import store from '@/store/modules/hots.js' +import tabStore from '@/store/modules/tabs.js' import {includeHeadersInData, hasAllColumnNames} from '@/frictionlessUtilities.js' import {allTablesAllColumnsFromSchema$} from '@/rxSubject.js' -async function initDataAndInferSchema(data) { +async function inferSchema(data) { const schema = await Schema.load({}) - await schema.infer(data) - return schema -} - -async function initDataAgainstSchema(data, schema) { - // provide schema rather than infer + // workaround for schema.infer stripping headers + let dataClone = [...data] + let headers = dataClone.shift() // frictionless default for csv dialect is that tables DO have headers - let table = await Table.load(data, {schema: schema, headers: 0}) - return table + // await schema.infer(data, {headers: 0}) + await schema.infer(dataClone, {headers: headers}) + return schema } function storeData(hotId, schema) { @@ -29,7 +28,7 @@ export async function guessColumnProperties() { let id = hot.guid let data = includeHeadersInData(hot) // let activeHot = HotRegister.getActiveHotIdData() - let schema = await initDataAndInferSchema(data) + let schema = await inferSchema(data) let isStored = storeData(id, schema) allTablesAllColumnsFromSchema$.next(store.getters.getAllHotTablesColumnProperties(store.state, store.getters)()) let message = isStored @@ -38,35 +37,75 @@ export async function guessColumnProperties() { return message } -function checkRow(rowNumber, row, schema, errorCollector) { +function checkRow(rowNumber, row, schema, tableRows, errorCollector) { + // if row contains foreign relation objects cast the original try { schema.castRow(row) } catch (err) { - if (err.multiple) { - for (const error of err.errors) { - let columnNumber = error.columnNumber || 'N/A' - errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: error.message, name: error.name}) - } - } else { - let columnNumber = err.columnNumber || 'N/A' - errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: err.message, name: err.name}) - } + errorHandler(err, rowNumber, errorCollector) } } -async function checkForSchema(data, hotId) { +async function buildSchema(data, hotId) { + let schema = await inferSchema(data) let hotTab = store.state.hotTabs[hotId] - let schema = await initDataAndInferSchema(data) schema.descriptor.fields = hotTab.columnProperties schema.descriptor.primaryKey = hotTab.tableProperties.primaryKeys schema.descriptor.foreignKeys = hotTab.tableProperties.foreignKeys - store.mutations.initMissingValues(store.state, store.state.hotTabs[hotId]) + store.mutations.initMissingValues(store.state, hotTab) schema.descriptor.missingValues = hotTab.tableProperties.missingValues - let table = await initDataAgainstSchema(data, schema) + return schema +} + +async function createFrictionlessTable(data, schema) { + // provide schema rather than infer + // frictionless default for csv dialect is that tables DO have headers + let dataClone = [...data] + let headers = dataClone.shift() + let table = await Table.load(dataClone, { + schema: schema, + headers: headers + }) table.schema.commit() return table } +async function collateForeignKeys(localHotId, callback) { + const foreignKeys = store.state.hotTabs[localHotId].tableProperties.foreignKeys + if (typeof foreignKeys === 'undefined') { + return false + } + let relations = {} + for (const foreignKey of foreignKeys) { + let foreignHotId = getHotIdFromForeignKeyForeignTable(foreignKey.reference.resource, localHotId) + // foreign keys must also have column properties set + if (!hasColumnProperties(foreignHotId, callback)) { + relations = false + break + } + let data = getForeignKeyData(foreignHotId) + let schema = await buildSchema(data, foreignHotId) + let table = await createFrictionlessTable(data, schema) + let rows = await table.read({keyed: true}) + relations[foreignKey.reference.resource] = rows + } + return relations +} + +function getForeignKeyData(foreignHotId) { + let hot = HotRegister.getInstance(foreignHotId) + return includeHeadersInData(hot) +} + +function getHotIdFromForeignKeyForeignTable(title, hotId) { + // check for fk in same table + if (title === '') { + return hotId + } + let tabId = tabStore.getters.findTabIdFromTitle(tabStore.state, tabStore.getters)(title) + return store.getters.getSyncHotIdFromTabId(store.state, store.getters)(tabId) +} + function isRowBlank(row) { return row.filter(Boolean).length === 0 } @@ -100,13 +139,63 @@ function checkHeaderErrors(headers, errorCollector, hasColHeaders) { } } +export async function validateActiveDataAgainstSchema(callback) { + let hot = HotRegister.getActiveInstance() + let hotId = hot.guid + if (!hasColumnProperties(hotId, callback)) { + return + } + const data = includeHeadersInData(hot) + const errorCollector = [] + const hasColHeaders = hot.hasColHeaders() + // ensure headers not lost from data + const headers = data[0] + checkHeaderErrors(headers, errorCollector, hasColHeaders) + let schema = await buildSchema(data, hotId) + let table = await createFrictionlessTable(data, schema) + // wait for frictionless pr#124 and uncomment + let relations = false + try { + relations = await collateForeignKeys(hotId, callback) + } catch (error) { + errorCollector.push({rowNumber: 0, + message: `There was a problem validating 1 or more foreign tables. Validate foreign tables first.`, + name: 'Invalid foreign table(s)' + }) + } + const stream = await table.iter({keyed: false, extended: true, stream: true, cast: true, forceCast: true, relations: relations}) + stream.on('data', (row) => { + // TODO: consider better way to accommodate or remove - need headers/column names so this logic may be redundant + let rowNumber = hasColHeaders + ? row[0] + : row[0] + 1 + if (row instanceof Error) { + errorHandler(row, rowNumber, errorCollector) + } else { + if (isRowBlank(row[2])) { + errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'}) + } + } + }) + stream.on('error', (error) => { + console.log(error) + const rowNumber = error.rowNumber ? error.rowNumber : 'N/A' + errorHandler(error, rowNumber, errorCollector) + // ensure error sent back + stream.end() + }) + stream.on('end', () => { + callback(errorCollector) + }) +} + function hasColumnProperties(hotId, callb) { let columnProperties = store.state.hotTabs[hotId].columnProperties if (!columnProperties || columnProperties.length === 0) { callb([ { rowNumber: 0, - message: `Column properties must be set.`, + message: `Column properties, including the column properties of any foreign keys, must be set.`, name: 'No Column Properties' } ]) @@ -116,7 +205,7 @@ function hasColumnProperties(hotId, callb) { callb([ { rowNumber: 0, - message: `Every Column property must have a 'name'.`, + message: `Every Column property, including the column properties of any foreign keys, must have a 'name'.`, name: 'Missing Column Property names' } ]) @@ -125,30 +214,14 @@ function hasColumnProperties(hotId, callb) { return true } -export async function validateActiveDataAgainstSchema(callback) { - let hot = HotRegister.getActiveInstance() - let id = hot.guid - if (!hasColumnProperties(id, callback)) { - return - } - let data = includeHeadersInData(hot) - const errorCollector = [] - const hasColHeaders = hot.hasColHeaders() - checkHeaderErrors(data[0], errorCollector, hasColHeaders) - let table = await checkForSchema(data, id) - // don't cast at stream, wait until row to cast otherwise not all errors will be reported. - const stream = await table.iter({extended: true, stream: true, cast: false, relations: true}) - stream.on('data', (row) => { - // TODO: consider better way to accommodate or remove - need headers/column names so this logic may be redundant - let rowNumber = hasColHeaders - ? row[0] - : row[0] + 1 - if (isRowBlank(row[2])) { - errorCollector.push({rowNumber: rowNumber, message: `Row ${rowNumber} is completely blank`, name: 'Blank Row'}) +function errorHandler(err, rowNumber, errorCollector) { + if (err.multiple) { + for (const error of err.errors) { + let columnNumber = error.columnNumber || 'N/A' + errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: error.message, name: error.name}) } - checkRow(rowNumber, row[2], table.schema, errorCollector) - }) - stream.on('end', () => { - callback(errorCollector) - }) + } else { + let columnNumber = err.columnNumber || 'N/A' + errorCollector.push({columnNumber: columnNumber, rowNumber: rowNumber, message: err.message, name: err.name}) + } } diff --git a/src/renderer/frictionlessDataPackage.js b/src/renderer/frictionlessDataPackage.js index f7b7de753..97c192e35 100644 --- a/src/renderer/frictionlessDataPackage.js +++ b/src/renderer/frictionlessDataPackage.js @@ -1,4 +1,4 @@ -import {Resource, Package, validate} from 'datapackage' +import {Resource, Package} from 'datapackage' import {HotRegister} from '@/hot.js' import tabStore from '@/store/modules/tabs.js' import hotStore from '@/store/modules/hots.js' @@ -22,7 +22,7 @@ export async function createDataPackage() { errorMessages.push('There is a problem with at least 1 package property. Please check and try again.') return errorMessages } - createZipFile(JSON.stringify(dataPackage.descriptor)) + createZipFile(dataPackage.descriptor) } } catch (err) { if (err) { @@ -60,6 +60,7 @@ function hasAllPackageRequirements(requiredMessages) { requiredMessages.push(`Package property, 'name' must be set.`) } addSourcesRequirements(packageProperties, requiredMessages, 'package') + addContributorsRequirements(packageProperties, requiredMessages, 'package') } return requiredMessages.length === 0 } @@ -131,7 +132,7 @@ function hasAllResourceRequirements(hot, requiredMessages) { requiredMessages.push(`Column properties must be set.`) } else { if (!hasAllColumnNames(hot.guid, columnProperties)) { - requiredMessages.push(`Column property names cannot be empty.`) + requiredMessages.push(`Column property names cannot be empty - set a Header Row`) } } return requiredMessages.length === 0 @@ -151,6 +152,30 @@ function addSourcesRequirements(properties, requiredMessages, entityName) { // console.log('source is valid') } } + if (properties.sources.length < 1) { + properties.sources = null + _.unset(properties, 'sources') + } +} + +function addContributorsRequirements(properties, requiredMessages, entityName) { + if (typeof properties.contributors === 'undefined') { + return + } + for (let contributor of properties.contributors) { + if (hasAllEmptyValues(contributor)) { + _.pull(properties.contributors, contributor) + } else if (!contributor.title || contributor.title.trim() === '') { + requiredMessages.push(`At least 1 ${entityName} contributor does not have a title.`) + return false + } else { + // console.log('contributor is valid') + } + } + if (properties.contributors.length < 1) { + properties.contributors = null + _.unset(properties, 'contributors') + } } function hasAllEmptyValues(propertyObject) { diff --git a/src/renderer/frictionlessUtilities.js b/src/renderer/frictionlessUtilities.js index 571bb0680..fcc38e8dd 100644 --- a/src/renderer/frictionlessUtilities.js +++ b/src/renderer/frictionlessUtilities.js @@ -11,6 +11,15 @@ export function includeHeadersInData(hot) { export function hasAllColumnNames(hotId, columnProperties) { let names = store.getters.getAllHotColumnNamesFromHotId(store.state, store.getters)(hotId) - let validNames = _.without(names, undefined, null, '') - return validNames.length === columnProperties.length + return hasAllValidColumnProperty(names, columnProperties) +} + +export function hasAllColumnTypes(hotId, columnProperties) { + let types = store.getters.getAllHotColumnTypesFromHotId(store.state, store.getters)(hotId) + return hasAllValidColumnProperty(types, columnProperties) +} + +function hasAllValidColumnProperty(values, columnProperties) { + let validValues = _.without(values, undefined, null, '') + return validValues.length === columnProperties.length } diff --git a/src/renderer/headerRow.js b/src/renderer/headerRow.js index 69870bfe1..9bbcd9de3 100644 --- a/src/renderer/headerRow.js +++ b/src/renderer/headerRow.js @@ -1,6 +1,6 @@ import {ipcRenderer as ipc} from 'electron' import {allTablesAllColumnNames$} from '@/rxSubject.js' -import store from '@/store/modules/hots.js' +import store from '@/store' import {HotRegister} from '@/hot.js' import {pushAllTabTitlesSubscription} from '@/store/modules/tabs.js' @@ -56,12 +56,12 @@ function updateHot(hot, data, header) { } function updateAllColumnsName(values) { - store.mutations.pushAllColumnsProperty(store.state, { + store.commit('pushAllColumnsProperty', { hotId: HotRegister.getActiveInstance().guid, key: 'name', values: values }) // do not allow getter to cache as does not seem to pick up change - allTablesAllColumnNames$.next(store.getters.getAllHotTablesColumnNames(store.state, store.getters)()) + allTablesAllColumnNames$.next(store.getters.getAllHotTablesColumnNames()) pushAllTabTitlesSubscription() } diff --git a/src/renderer/importPackage.js b/src/renderer/importPackage.js index ff8b41932..a90fa4461 100644 --- a/src/renderer/importPackage.js +++ b/src/renderer/importPackage.js @@ -1,14 +1,16 @@ import fs from 'fs-extra' import path from 'path' -import hotStore from '@/store/modules/hots.js' -import tabStore from '@/store/modules/tabs.js' +import store from '@/store' import unzipper from 'unzipper' import etl from 'etl' import {ipcRenderer as ipc} from 'electron' +import {Resource, Package} from 'datapackage' export async function unzipFile(zipSource, storeCallback) { try { - let processedProperties = await unzipFileToDir(zipSource, createUnzipDestination(zipSource)) + let destination = createUnzipDestination(zipSource) + await fs.ensureDir(destination) + let processedProperties = await unzipFileToDir(zipSource, destination) storeCallback(processedProperties) } catch (err) { console.log(`Error processing zip source: ${zipSource}`, err) @@ -22,24 +24,36 @@ function createUnzipDestination(zipSource) { } async function unzipFileToDir(zipSource, unzipDestination) { - let processed = {json: [], csv: [], md: []} + let processed = {json: [], resource: [], md: []} await fs.createReadStream(zipSource).pipe(unzipper.Parse()).pipe(etl.map(async entry => { let fileDestination = `${unzipDestination}/${entry.path}` await processStream(entry, processed, fileDestination) })).promise() validateMdFile(processed) // wait until all tabs opened before processing data package json - let processedProperties = await processJsonFile(processed, unzipDestination) + if (processed.json.length !== 1) { + throw new Error('There must be 1, and only 1, json file.') + } + let dataPackageJson = await getDataPackageJson(processed) + let csvPathHotIds = await processResources(dataPackageJson, unzipDestination, processed) + let processedProperties = await processJson(dataPackageJson, csvPathHotIds, unzipDestination) return processedProperties } +async function getDataPackageJson(processed) { + let filename = processed.json[0] + let text = await stringify(filename) + let dataPackageJson = JSON.parse(text) + return dataPackageJson +} + async function processStream(entry, processed, fileDestination) { switch (path.extname(entry.path)) { case '.csv': + case '.tsv': await fs.ensureFile(fileDestination) await unzippedEntryToFile(entry, fileDestination) - await ipc.send('openFileIntoTab', fileDestination) - processed.csv.push(entry.path) + processed.resource.push(entry.path) break case '.json': await fs.ensureFile(fileDestination) @@ -51,7 +65,7 @@ async function processStream(entry, processed, fileDestination) { await fs.ensureFile(fileDestination) await unzippedEntryToFile(entry, fileDestination) let textMd = await stringify(fileDestination) - await setProvenance(textMd) + setProvenance(textMd) processed.md.push(fileDestination) break default: @@ -83,21 +97,35 @@ async function stringify(filename) { return text } -async function setProvenance(text) { - hotStore.mutations.pushProvenance(hotStore.state, text) +function setProvenance(text) { + store.commit('pushProvenance', text) } -async function processJsonFile(processed, unzipDestination) { - if (processed.json.length !== 1) { - throw new Error('There must be 1, and only 1, json file.') - } +async function processResources(dataPackageJson, unzipDestination, processed) { + let resourcePaths = await getAllResourcePaths(dataPackageJson, unzipDestination) let csvPathHotIds = await getHotIdsFromFilenames(processed, unzipDestination) - let filename = processed.json[0] - let text = await stringify(filename) - let datapackageJson = JSON.parse(text) - validateResourcesAndDataFiles(getAllResourcePaths(datapackageJson), _.keys(csvPathHotIds)) - let processedProperties = processJson(datapackageJson, csvPathHotIds) - return processedProperties + validateResourcesAndDataFiles(resourcePaths, _.keys(csvPathHotIds)) + return csvPathHotIds +} + +async function getAllResourcePaths(dataPackageJson, unzipDestination) { + let resourcePaths = [] + for (let dataResource of dataPackageJson.resources) { + let fileDestination = `${unzipDestination}/${dataResource.path}` + let format = dataResourcetoFormat(dataResource) + await ipc.send('openFileIntoTab', fileDestination, format) + resourcePaths.push(dataResource.path) + } + return resourcePaths +} + +function dataResourcetoFormat(dataResource) { + let format = {} + _.assign(format, dataResource) + for (const key of ['missingValues', 'name', 'path', 'profile', 'schema']) { + _.unset(format, key) + } + return format } function validateResourcesAndDataFiles(resourcePaths, csvPaths) { @@ -111,14 +139,14 @@ function validateResourcesAndDataFiles(resourcePaths, csvPaths) { async function getHotIdsFromFilenames(processed, unzipDestination) { let dataPackageJson = processed.json[0] let csvTabs = {} - for (let pathname of processed.csv) { + for (let pathname of processed.resource) { let fileDestination = `${unzipDestination}/${pathname}` let tabId = await getTabIdFromFilename(fileDestination) // every processed csv should have a matching tab if (!tabId) { throw new Error(`There was a problem matching ${fileDestination} with an opened tab.`) } - let hotId = _.findKey(hotStore.state.hotTabs, {tabId: tabId}) + let hotId = _.findKey(store.getters.getHotTabs, {tabId: tabId}) // ensure csv path accounts for parent folders zipped up let re = new RegExp('^' + processed.parentFolders + '/') let resourcePathname = _.replace(pathname, re, '') @@ -129,11 +157,11 @@ async function getHotIdsFromFilenames(processed, unzipDestination) { async function getTabIdFromFilename(filename) { return new Promise((resolve, reject) => { - let tabId = _.findKey(tabStore.state.tabObjects, {filename: filename}) + let tabId = _.findKey(store.getters.getTabObjects, {filename: filename}) if (!tabId) { // wait for tabs to be ready _.delay(function(filename) { - resolve(_.findKey(tabStore.state.tabObjects, {filename: filename})) + resolve(_.findKey(store.getters.getTabObjects, {filename: filename})) }, 500, filename) } else { resolve(tabId) @@ -141,15 +169,8 @@ async function getTabIdFromFilename(filename) { }) } -function getAllResourcePaths(datapackageJson) { - let resourcePaths = datapackageJson.resources.map(function(dataResource) { - return dataResource.path - }) - return resourcePaths -} - -function processJson(datapackageJson, csvPathHotIds) { - let allTableProperties = datapackageJson.resources +function processJson(dataPackageJson, csvPathHotIds, unzipDestination) { + let allTableProperties = dataPackageJson.resources let allColumnPropertiesByHotId = {} let allTablePropertiesByHotId = {} for (let tableProperties of allTableProperties) { @@ -162,9 +183,9 @@ function processJson(datapackageJson, csvPathHotIds) { _.unset(tableProperties, 'schema') allTablePropertiesByHotId[csvPathHotIds[tableProperties.path]] = tableProperties } - _.unset(datapackageJson, 'resources') + _.unset(dataPackageJson, 'resources') return { - package: datapackageJson, + package: dataPackageJson, tables: allTablePropertiesByHotId, columns: allColumnPropertiesByHotId } diff --git a/src/renderer/index.js b/src/renderer/index.js index f32b09f91..d3457bab5 100644 --- a/src/renderer/index.js +++ b/src/renderer/index.js @@ -19,10 +19,10 @@ export function addHotContainerListeners(container) { var f = e.dataTransfer.files[0] fs.readFile(f.path, 'utf-8', function(err, data) { if (err) { - console.log(err.stack) + console.log(err) } // if we're dragging a file in, default the format to comma-separated - loadData(hot, data, file.formats.csv.options) + loadData(hot, data, file.formats.csv) }) } @@ -48,11 +48,12 @@ ipc.on('getCSV', function(e, format) { let hot = HotRegister.getActiveInstance() var data // if no format specified, default to csv - if (typeof format === 'undefined') { - data = $.csv.fromArrays(hot.getData()) - } else { - data = $.csv.fromArrays(hot.getData(), format.options) - } + // TODO: update to node csv and check dialect mappings + // if (typeof format === 'undefined') { + // data = $.csv.fromArrays(hot.getData()) + // } else { + // data = $.csv.fromArrays(hot.getData(), format.options) + // } ipc.send('sendCSV', data) }) diff --git a/src/renderer/mixins/PackageTooltip.vue b/src/renderer/mixins/PackageTooltip.vue index bb458f488..17c770114 100644 --- a/src/renderer/mixins/PackageTooltip.vue +++ b/src/renderer/mixins/PackageTooltip.vue @@ -33,6 +33,10 @@ export default { 'tooltipPackageLicenses': { components: {externalLink}, template: `
The under which this data package is provided
` + }, + 'tooltipPackageContributors': { + components: {externalLink}, + template: `
Each must have a title and may contain path, email, role and organization
` } } } diff --git a/src/renderer/mixins/RelationKeys.vue b/src/renderer/mixins/RelationKeys.vue index 7115a56af..50190c0b4 100644 --- a/src/renderer/mixins/RelationKeys.vue +++ b/src/renderer/mixins/RelationKeys.vue @@ -21,7 +21,7 @@ export default { } }, computed: { - ...mapGetters(['getActiveTab', 'getAllHotTablesColumnNames', 'getAllTabTitles', 'getTabObjects', 'getHotIdFromTabId', 'getTabId', 'getAllForeignKeys', 'tabTitle']) + ...mapGetters(['getActiveTab', 'getAllHotTablesColumnNames', 'getAllTabTitles', 'getTabObjects', 'getHotIdFromTabId', 'getSyncHotIdFromTabId', 'getAllForeignKeys', 'tabTitle']) }, methods: { ...mapMutations(['pushForeignKeysLocalFieldsForTable', 'pushForeignKeysForeignFieldsForTable', 'pushForeignKeysForeignTableForTable']), diff --git a/src/renderer/mixins/ValidationRules.vue b/src/renderer/mixins/ValidationRules.vue index 1e6d72cf5..170190df3 100644 --- a/src/renderer/mixins/ValidationRules.vue +++ b/src/renderer/mixins/ValidationRules.vue @@ -45,7 +45,7 @@ export default { regex: 'The name field format is invalid. It must consist only of lowercase alphanumeric characters plus ".", "-" and "_".' }, enum: { - required: 'Separate valid values with a comma.' + required: 'Quote each "valid value" and separate with a comma.' }, pattern: { required: 'There must be a pattern present.' diff --git a/src/renderer/partials/About.vue b/src/renderer/partials/About.vue index 5485d226c..055ad2610 100644 --- a/src/renderer/partials/About.vue +++ b/src/renderer/partials/About.vue @@ -46,7 +46,7 @@ export default { { items: [ { - label: `Beta version ${this.getApplicationVersion()} - access the support forum or report issues via the help menu` + label: `Beta version ${this.getApplicationVersion()} - Access the support forum or report issues via the Help menu` } ] }, @@ -82,6 +82,16 @@ export default { label: 'Includes software developed by the Queensland Cyber Infrastructure Foundation on behalf of the Queensland Government and the ODI Australian Network' } ] + }, + { + items: [{ + image: 'static/img/frictionless-data.png', + link: 'https://frictionlessdata.io' + }, + { + label: 'Using Frictionless Data specifications and software' + } + ] } ] } diff --git a/src/renderer/partials/ColumnProperties.vue b/src/renderer/partials/ColumnProperties.vue index 9f5c8f50f..78bd9f940 100644 --- a/src/renderer/partials/ColumnProperties.vue +++ b/src/renderer/partials/ColumnProperties.vue @@ -314,6 +314,10 @@ export default { return true }, setConstraintValue: function(key, value) { + // TODO: split at double quote and comma + // if (key === 'enum') { + // value = _.toArray(value) + // } this.constraintInputKeyValues[key] = value this.pushConstraintInputKeyValues() }, diff --git a/src/renderer/partials/Contributors.vue b/src/renderer/partials/Contributors.vue new file mode 100644 index 000000000..46de47e7d --- /dev/null +++ b/src/renderer/partials/Contributors.vue @@ -0,0 +1,137 @@ + + + + diff --git a/src/renderer/partials/ForeignKeys.vue b/src/renderer/partials/ForeignKeys.vue index 907b7c1c4..c44a958d2 100644 --- a/src/renderer/partials/ForeignKeys.vue +++ b/src/renderer/partials/ForeignKeys.vue @@ -157,7 +157,7 @@ export default { let tabId = _.findKey(this.allTabTableNames, function(o) { return o === tableName }) - let hotId = this.getTabId(tabId) + let hotId = this.getSyncHotIdFromTabId(tabId) return hotId }, getSelectedLocalKeys: function(index) { diff --git a/src/renderer/partials/Licenses.vue b/src/renderer/partials/Licenses.vue index 65f6a3760..ccea786b9 100644 --- a/src/renderer/partials/Licenses.vue +++ b/src/renderer/partials/Licenses.vue @@ -69,7 +69,6 @@ export default { 'path': 'https://data.gov.tw/license/' }, { - 'name': 'pdm', 'title': 'Public Domain Mark', 'path': 'http://creativecommons.org/publicdomain/mark/1.0/' }], diff --git a/src/renderer/partials/PackageProperties.vue b/src/renderer/partials/PackageProperties.vue index 82ce8850e..51ccaa936 100644 --- a/src/renderer/partials/PackageProperties.vue +++ b/src/renderer/partials/PackageProperties.vue @@ -18,6 +18,7 @@ import SideNav from '@/partials/SideNav' import licenses from '@/partials/Licenses' import sources from '@/partials/Sources' +import contributors from '@/partials/Contributors' import PackageTooltip from '@/mixins/PackageTooltip' import ValidationRules from '@/mixins/ValidationRules' import { @@ -31,35 +32,32 @@ export default { mixins: [ValidationRules, PackageTooltip], components: { licenses, - sources + sources, + contributors }, data() { return { formprops: [{ label: 'Name*', key: 'name', - type: 'input', tooltipId: 'tooltip-package-name', tooltipView: 'tooltipPackageName' }, { label: 'Id', key: 'id', - type: 'input', tooltipId: 'tooltip-package-id', tooltipView: 'tooltipPackageId' }, { label: 'Title', key: 'title', - type: 'input', tooltipId: 'tooltip-package-title', tooltipView: 'tooltipPackageTitle' }, { label: 'Description', key: 'description', - type: 'markdown', tooltipId: 'tooltip-package-description', tooltipView: 'tooltipPackageDescription' }, @@ -67,14 +65,12 @@ export default { { label: 'Version', key: 'version', - type: 'input', tooltipId: 'tooltip-package-version', tooltipView: 'tooltipPackageVersion' }, { label: 'Source(s)', key: 'sources', - type: 'dropdown', tooltipId: 'tooltip-package-sources', tooltipView: 'tooltipPackageSources' }, @@ -83,6 +79,12 @@ export default { key: 'licenses', tooltipId: 'tooltip-package-licenses', tooltipView: 'tooltipPackageLicenses' + }, + { + label: 'Contributor(s)', + key: 'contributors', + tooltipId: 'tooltip-package-contributors', + tooltipView: 'tooltipPackageContributors' } ] } diff --git a/src/renderer/partials/ProvenanceProperties.vue b/src/renderer/partials/ProvenanceProperties.vue index ba9bf8ec7..bf36fb4d8 100644 --- a/src/renderer/partials/ProvenanceProperties.vue +++ b/src/renderer/partials/ProvenanceProperties.vue @@ -63,19 +63,22 @@ export default { return { isPreview: false, provenance: '', - placeholder: `### Introduction + placeholder: `Short description of the dataset (the first sentence and first paragraph should be extractable to provide short standalone descriptions) -### Why was the dataset created? (reference legislation if relevant) +### Why was the dataset created? +reference legislation if relevant -### How was it collected - what events lead up to its collection? +### How was it collected +what events lead up to its collection? -### When was it collected? (Temporal extent) +### When was it collected? -### Where was it collected? (Spatial extent name, coordinate reference system, minimum bounding rectangle) +### Where was it collected? ### Which instruments were used to collect it? -### What does “null” mean? Unknown, missing or not applicable? +### What does “null” mean? +are null values unknown, missing or not applicable? ### Other comments diff --git a/src/renderer/partials/SideNav.vue b/src/renderer/partials/SideNav.vue index 38764c14c..208802b73 100644 --- a/src/renderer/partials/SideNav.vue +++ b/src/renderer/partials/SideNav.vue @@ -33,7 +33,7 @@ export default { }, methods: { isSharedComponent: function(key) { - let isShared = ['sources', 'licenses', 'primaryKeys', 'foreignKeys'].indexOf(key) !== -1 + let isShared = ['sources', 'licenses', 'primaryKeys', 'foreignKeys', 'contributors'].indexOf(key) !== -1 return isShared }, propertyGetObjectGivenHotId: function(key, hotId) { diff --git a/src/renderer/partials/Sources.vue b/src/renderer/partials/Sources.vue index 288c80af1..4c63ea74b 100644 --- a/src/renderer/partials/Sources.vue +++ b/src/renderer/partials/Sources.vue @@ -10,7 +10,7 @@ - @@ -54,6 +54,7 @@ export default { getSources: { async get() { let tab = this.getActiveTab + // TODO: may need distinction here for package vs tables let sources = await this.getSourcesFromTab(tab) return sources }, @@ -86,14 +87,9 @@ export default { let sources = this.getPropertyGivenHotId('sources', hotId) return sources }, + // TODO: fix this redundant method initSources: async function(tab) { let sources = await this.getSourcesFromTab(tab) - if (!sources) { - const vueAddSource = this.addSource - _.delay(function() { - vueAddSource() - }, 100) - } }, setSourceProp: function(index, prop, value) { this.setProperty(`sources[${index}][${prop}]`, value) diff --git a/src/renderer/store/modules/hots.js b/src/renderer/store/modules/hots.js index 77ac9e308..c63bdee1e 100644 --- a/src/renderer/store/modules/hots.js +++ b/src/renderer/store/modules/hots.js @@ -26,12 +26,6 @@ export function getHotIdFromTabIdFunction() { return getters.getHotIdFromTabId(state, getters) } -function mergeSchemaForColumnProperties(currentProperties, descriptor) { - let properties = [...currentProperties] - _.merge(properties, descriptor.fields) - return properties -} - const getters = { getHotTabs: state => { return state.hotTabs @@ -59,15 +53,23 @@ const getters = { return hotIdColumnNames }, getAllHotColumnNamesFromHotId: (state, getters) => (hotId) => { + return getters.getAllHotColumnPropertyFromHotId(state, getters)({hotId: hotId, key: 'name'}) + }, + getAllHotColumnTypesFromHotId: (state, getters) => (hotId) => { + return getters.getAllHotColumnPropertyFromHotId(state, getters)({hotId: hotId, key: 'type'}) + }, + getAllHotColumnPropertyFromHotId: (state, getters) => (property) => { + const hotId = property.hotId + const propertyKey = property.key if (!state.hotTabs[hotId].columnProperties) { state.hotTabs[hotId].columnProperties = [] // return } - let names = state.hotTabs[hotId].columnProperties.map(column => { - let name = column.name - return column.name + let values = state.hotTabs[hotId].columnProperties.map(column => { + let value = column[propertyKey] + return column[propertyKey] }) - return names + return values }, getHotIdFromTabId: (state, getters) => (tabId) => { return new Promise((resolve, reject) => { @@ -82,7 +84,7 @@ const getters = { } }) }, - getTabId: (state, getters) => (tabId) => { + getSyncHotIdFromTabId: (state, getters) => (tabId) => { let hotId = _.findKey(state.hotTabs, {tabId: tabId}) return hotId }, @@ -100,6 +102,9 @@ const getters = { getPackageProperty: (state, getters) => (property) => { return state.packageProperties[property.key] }, + getPackageProperties: state => { + return state.packageProperties + }, getConstraint: (state, getters) => (property) => { let hotColumnProperties = getHotColumnPropertiesFromPropertyObject(property) let constraints = hotColumnProperties['constraints'] @@ -204,7 +209,14 @@ const mutations = { let hotTab = state.hotTabs[hotId] mutations.initColumnProperties(state, hotTab) // we cannot mutate the vuex state itself (in lodash call) - we can only assign a new value - state.hotTabs[hotId].columnProperties = mergeSchemaForColumnProperties(hotTab.columnProperties, hotIdSchema.schema.descriptor) + let columnProperties = [] + for (let column of hotTab.columnProperties) { + let nextObject = {} + columnProperties.push(nextObject) + _.assign(nextObject, column) + } + _.merge(columnProperties, hotIdSchema.schema.descriptor.fields) + state.hotTabs[hotId].columnProperties = columnProperties return state.hotTabs[hotId].columnProperties }, initColumnProperties(state, hotTab) { diff --git a/src/renderer/store/modules/tabs.js b/src/renderer/store/modules/tabs.js index b96c6c412..ad14f5506 100644 --- a/src/renderer/store/modules/tabs.js +++ b/src/renderer/store/modules/tabs.js @@ -48,6 +48,9 @@ const getters = { allTabTitles[tabId] = object.title }) return allTabTitles + }, + findTabIdFromTitle: (state, getters) => (title) => { + return _.findKey(state.tabObjects, function(o) { return o.title === title }) } } diff --git a/static/css/columnprops.styl b/static/css/columnprops.styl index 0caeae89c..54390fa1a 100644 --- a/static/css/columnprops.styl +++ b/static/css/columnprops.styl @@ -17,6 +17,7 @@ dangerColor = #ff3860 width 70% div#constraints display block + margin-top 15px padding-right 0 padding-top 5px div diff --git a/static/css/contributors.styl b/static/css/contributors.styl new file mode 100644 index 000000000..b2bb37067 --- /dev/null +++ b/static/css/contributors.styl @@ -0,0 +1,35 @@ +#contributors + margin-bottom 10px + .contributor + margin-right 0 + padding-right 0 + padding-left 0 + margin-bottom 5px + display block + width 100% + .inputs-container + display inline-block + width 90% + float left + .input-group + width 100% + input + width 100% + padding-left 10px !important + .input-group-addon + width 30% + text-align left + button + display inline-block + height 100% !important + min-height 100% !important + margin 30px 0px 30px 5px + padding 5px 8px 5px 8px + float right + .button-container + width 100% + overflow hidden + display block + button.add-contributor + display block + float right diff --git a/static/img/frictionless-data.png b/static/img/frictionless-data.png new file mode 100644 index 000000000..4add35490 Binary files /dev/null and b/static/img/frictionless-data.png differ diff --git a/yarn.lock b/yarn.lock index a362710a1..41ed4d59f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -491,6 +491,13 @@ axios@^0.16.1: follow-redirects "^1.2.3" is-buffer "^1.1.5" +axios@^0.17.1: + version "0.17.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.17.1.tgz#2d8e3e5d0bdbd7327f91bc814f5c57660f81824d" + dependencies: + follow-redirects "^1.2.5" + is-buffer "^1.1.5" + babel-code-frame@^6.22.0, babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" @@ -3142,9 +3149,9 @@ electron-chromedriver@~1.7.1: electron-download "^4.1.0" extract-zip "^1.6.5" -electron-debug@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/electron-debug/-/electron-debug-1.5.0.tgz#d88c02146efb7fc5ae1b21eac56fbe4987eae50c" +electron-debug@~1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/electron-debug/-/electron-debug-1.4.0.tgz#bec7005522220a9d0622153352e1bbff0f37af2e" dependencies: electron-is-dev "^0.3.0" electron-localshortcut "^3.0.0" @@ -4045,6 +4052,12 @@ follow-redirects@^1.2.3: dependencies: debug "^3.1.0" +follow-redirects@^1.2.5: + version "1.4.1" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.4.1.tgz#d8120f4518190f55aac65bb6fc7b85fcd666d6aa" + dependencies: + debug "^3.1.0" + for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -5451,10 +5464,6 @@ jsonfile@^4.0.0: optionalDependencies: graceful-fs "^4.1.6" -jsonfile@~1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-1.1.1.tgz#da4fd6ad77f1a255203ea63c7bc32dc31ef64433" - jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -6240,10 +6249,6 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@0.3.x: - version "0.3.5" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7" - mkdirp@0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.0.tgz#1d73076a6df986cd9344e15e71fcc05a4c9abf12" @@ -6385,10 +6390,6 @@ ncname@1.0.x: dependencies: xml-char-classes "^1.0.0" -ncp@~0.4.2: - version "0.4.2" - resolved "https://registry.yarnpkg.com/ncp/-/ncp-0.4.2.tgz#abcc6cbd3ec2ed2a729ff6e7c1fa8f01784a8574" - ndjson@^1.4.0: version "1.5.0" resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8" @@ -6429,15 +6430,6 @@ node-forge@0.6.33: version "0.6.33" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.6.33.tgz#463811879f573d45155ad6a9f43dc296e8e85ebc" -node-fs-extra@^0.8.2: - version "0.8.2" - resolved "https://registry.yarnpkg.com/node-fs-extra/-/node-fs-extra-0.8.2.tgz#09fb2b60d30f7d703e361ecb626a91404f17097a" - dependencies: - jsonfile "~1.1.0" - mkdirp "0.3.x" - ncp "~0.4.2" - rimraf "~2.2.0" - node-gyp@^3.6.2: version "3.6.2" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.6.2.tgz#9bfbe54562286284838e750eac05295853fa1c60" @@ -8028,7 +8020,7 @@ rimraf@2, rimraf@^2.5.1, rimraf@^2.5.2, rimraf@^2.5.4, rimraf@^2.6.0, rimraf@^2. dependencies: glob "^7.0.5" -rimraf@^2.2.8, rimraf@~2.2.0, rimraf@~2.2.6: +rimraf@^2.2.8, rimraf@~2.2.6: version "2.2.8" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" @@ -8877,11 +8869,11 @@ tableschema@^1.2.1: stream-to-async-iterator "^0.2.0" tv4 "^1.2.7" -tableschema@^1.5.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.5.1.tgz#cf030be7ac8dd7802f6a86e9826e34eaf59512a4" +tableschema@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/tableschema/-/tableschema-1.7.0.tgz#79d8ab6cc0a45ddf57779fe8ff32f1cdd63d471e" dependencies: - axios "^0.16.1" + axios "^0.17.1" csv "^1.1.1" d3-time-format "^0.3.2" es6-error "^4.0.2" @@ -8890,6 +8882,7 @@ tableschema@^1.5.1: regenerator-runtime "^0.11.0" stream-to-async-iterator "^0.2.0" tv4 "^1.2.7" + validator "^9.2.0" tapable@^0.2.7: version "0.2.8" @@ -9516,6 +9509,10 @@ validate.io-undefined@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/validate.io-undefined/-/validate.io-undefined-1.0.3.tgz#7e27fcbb315b841e78243431897671929e20b7f4" +validator@^9.2.0: + version "9.4.0" + resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.0.tgz#c503ef88f7e6b8fb7688599267b309482d81ae60" + validator@~7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/validator/-/validator-7.0.0.tgz#c74deb8063512fac35547938e6f0b1504a282fd2"