Skip to content

Commit

Permalink
#988, #1003: Enable writing out of custom names and values for packag…
Browse files Browse the repository at this point in the history 
…e, table and columns on export. Ensure custom names are not reserved by current package,table or column properties.
  • Loading branch information
mattRedBox committed Jun 16, 2020
1 parent 9c819c7 commit 260dd63
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 60 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
"csv-sniffer": "^0.1.1",
"datapackage": "~1.1.9",
"detect-newline": "^3.0.0",
"electron-devtools-installer": "^3.0.0",
"electron-settings": "^3.2.0",
"escape-regexp": "^0.0.1",
"etl": "^0.5.12",
Expand Down Expand Up @@ -170,7 +171,6 @@
"electron": "^2.0.12",
"electron-builder": "^21.2.0",
"electron-debug": "^1.5.0",
"electron-devtools-installer": "^2.2.4",
"eslint": "^5.7.0",
"eslint-config-standard": "^12.0.0",
"eslint-formatter-friendly": "^6.0.0",
Expand Down
94 changes: 53 additions & 41 deletions src/renderer/frictionlessDataPackage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,16 @@ function hasAllPackageRequirements (requiredMessages) {
requiredMessages.push(`Provenance properties must be set.`)
}
let packageProperties = hotStore.state.packageProperties
if (!packageProperties || _.isEmpty(packageProperties)) {
if (_.isEmpty(packageProperties)) {
requiredMessages.push(`Package properties must be set.`)
} else {
let name = packageProperties.name
if (!name || name.trim() === '') {
let name = _.get(packageProperties, 'name', '').trim()
if (_.isEmpty(name)) {
requiredMessages.push(`Package property, 'name' must be set.`)
}
addSourcesRequirements(packageProperties, requiredMessages, 'package')
addContributorsRequirements(packageProperties, requiredMessages, 'package')
addRequirementsForPropertyList(packageProperties, requiredMessages, 'package', 'sources')
addRequirementsForPropertyList(packageProperties, requiredMessages, 'package', 'contributors')
checkReservedWordsForPropertyList(packageProperties, requiredMessages, 'package', 'customs')
}
return requiredMessages.length === 0
}
Expand All @@ -75,6 +76,7 @@ function addPackageProperties (descriptor) {
let packageProperties = hotStore.state.packageProperties
_.merge(descriptor, packageProperties)
removeEmptiesFromDescriptor(descriptor)
updateCustomsForProperties(descriptor, 'package')
}

async function buildAllResourcesForDataPackage (dataPackage, errorMessages) {
Expand Down Expand Up @@ -117,76 +119,70 @@ async function createValidResource (hotId, errorMessages) {
}

function hasAllResourceRequirements (hot, requiredMessages) {
let tableProperties = hotStore.state.hotTabs[hot.guid].tableProperties
let tableProperties = _.cloneDeep(hotStore.state.hotTabs[hot.guid].tableProperties)
if (!tableProperties) {
requiredMessages.push(`Table properties must be set.`)
} else {
let name = tableProperties.name
if (!name || name.trim() === '') {
requiredMessages.push(`Table property, 'name', must not be empty.`)
}
addSourcesRequirements(tableProperties, requiredMessages, 'table')
addRequirementsForPropertyList(tableProperties, requiredMessages, 'table', 'sources')
addForeignKeyRequirements(tableProperties, requiredMessages)
checkReservedWordsForPropertyList(tableProperties, requiredMessages, 'table', 'customs')
}
let columnProperties = hotStore.state.hotTabs[hot.guid].columnProperties
let columnProperties = _.cloneDeep(hotStore.state.hotTabs[hot.guid].columnProperties)
if (!columnProperties) {
requiredMessages.push(`Column properties must be set.`)
} else {
let names = getValidNames(hot.guid)
if (!hasAllColumnNames(hot.guid, columnProperties, names)) {
requiredMessages.push(`Column property names cannot be empty - set a Header Row`)
}
for (const nextColumn of columnProperties) {
console.dir(nextColumn)
checkReservedWordsForPropertyList(nextColumn, requiredMessages, 'column', 'customs')
}
}
return requiredMessages.length === 0
}

function addSourcesRequirements (properties, requiredMessages, entityName) {
if (typeof properties.sources === 'undefined') {
function addRequirementsForPropertyList (properties, requiredMessages, entityName, propertyName, requiredAttribute = 'title') {
const requirementsAsList = _.get(properties, propertyName)
if (!_.isArray(requirementsAsList)) {
return
}
for (let source of properties.sources) {
if (hasAllEmptyValues(source)) {
_.pull(properties.sources, source)
} else if (!source.title || source.title.trim() === '') {
requiredMessages.push(`At least 1 ${entityName} source does not have a title.`)
return false
for (let property of requirementsAsList) {
if (hasAllEmptyValues(property)) {
_.pull(requirementsAsList, property)
} else {
// console.log('source is valid')
if (_.isEmpty(_.get(property, requiredAttribute, '').trim())) {
requiredMessages.push(`At least 1 of ${entityName} ${propertyName} does not have a ${requiredAttribute}.`)
return false
}
}
}
if (properties.sources.length < 1) {
properties.sources = null
_.unset(properties, 'sources')
}
}

function addContributorsRequirements (properties, requiredMessages, entityName) {
if (typeof properties.contributors === 'undefined') {
function checkReservedWordsForPropertyList (properties, requiredMessages, entityName, propertyName, requiredAttribute = 'name') {
const requirementsAsList = _.get(properties, propertyName)
let reserved = _.keys(properties)
if (!_.isArray(requirementsAsList)) {
return
}
for (let contributor of properties.contributors) {
if (hasAllEmptyValues(contributor)) {
_.pull(properties.contributors, contributor)
} else if (_.isEmpty(_.get(contributor, 'title')) || contributor.title.trim() === '') {
requiredMessages.push(`At least 1 ${entityName} contributor does not have a title.`)
return false
} else {
// console.log('contributor is valid')
for (let property of requirementsAsList) {
const toMatch = _.get(property, requiredAttribute)
if (_.includes(reserved, toMatch)) {
requiredMessages.push(`${_.capitalize(entityName)} already uses: '${toMatch}', so it cannot be used again in ${entityName} '${propertyName}' properties.`)
}
}
if (properties.contributors.length < 1) {
properties.contributors = null
_.unset(properties, 'contributors')
}
}

function hasAllEmptyValues (propertyObject) {
let isEmpty = true
_.forOwn(propertyObject, function (value, key) {
if (value.trim().length > 0) {
isEmpty = false
return false
}
isEmpty = _.isEmpty(_.trim(value))
return isEmpty
})
return isEmpty
}
Expand All @@ -205,7 +201,7 @@ function addForeignKeyRequirements (tableProperties, requiredMessages) {

async function buildResource (tabId, hotId) {
let resource = await initResourceAndInfer()
let descriptor = resource.descriptor
let descriptor = _.cloneDeep(resource.descriptor)
addColumnProperties(descriptor, hotId)
addTableProperties(descriptor, hotId)
removeEmptiesFromDescriptor(descriptor)
Expand All @@ -225,12 +221,16 @@ function addColumnProperties (descriptor, hotId) {
let columnProperties = hotStore.state.hotTabs[hotId].columnProperties
descriptor.schema = {}
descriptor.schema.fields = columnProperties
for (const field of descriptor.schema.fields) {
updateCustomsForProperties(field, 'column')
}
}

function addTableProperties (descriptor, hotId) {
let tableProperties = hotStore.state.hotTabs[hotId].tableProperties
_.merge(descriptor, tableProperties)
moveTableSchemaProperties(descriptor, tableProperties)
updateCustomsForProperties(descriptor, 'table')
}

function moveTableSchemaProperties (descriptor, tableProperties) {
Expand All @@ -247,6 +247,7 @@ function moveTableSchemaProperties (descriptor, tableProperties) {
function removeEmptiesFromDescriptor (descriptor) {
removeEmpty(descriptor, 'licenses')
removeEmpty(descriptor, 'sources')
removeEmpty(descriptor, 'customs')
}

function removeNonFrictionlessKeys (descriptor) {
Expand All @@ -256,7 +257,7 @@ function removeNonFrictionlessKeys (descriptor) {
}

function removeEmpty (descriptor, propertyName) {
if (descriptor[propertyName] && descriptor[propertyName].length === 0) {
if (_.isEmpty(_.get(descriptor, propertyName))) {
_.unset(descriptor, propertyName)
}
}
Expand All @@ -269,3 +270,14 @@ function addPath (descriptor, tabId) {
// resource paths must be POSIX https://frictionlessdata.io/specs/data-resource/#url-or-path
descriptor.path = _.replace(osPath, '\\', '/')
}

function updateCustomsForProperties (descriptor, customType) {
let customs = _.get(descriptor, 'customs', [])
_.unset(descriptor, 'customs')
do {
const custom = customs.pop()
if (!_.isEmpty(_.get(custom, 'name', '')) && !_.isEmpty(_.get(custom, 'value', '')) && _.includes(_.get(custom, 'types', []), customType)) {
_.set(descriptor, custom.name, custom.value)
}
} while (!_.isEmpty(customs))
}
61 changes: 43 additions & 18 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,6 @@
resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-5.0.3.tgz#bc5b5532ecafd923a61f2fb097e3b108c0106a3f"
integrity sha512-GLyWIFBbGvpKPGo55JyRZAo4lVbnBiD52cKlw/0Vt+wnmKvWJkpZvsjVoaIolyBXDeAQKSicRtqFNPem9w0WYA==

"[email protected]":
version "0.0.6"
resolved "https://registry.yarnpkg.com/7zip/-/7zip-0.0.6.tgz#9cafb171af82329490353b4816f03347aa150a30"
integrity sha1-nK+xca+CMpSQNTtIFvAzR6oVCjA=

"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.8.3":
version "7.8.3"
resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.8.3.tgz#33e25903d7481181534e12ec0a25f16b6fcf419e"
Expand Down Expand Up @@ -3431,11 +3426,6 @@ cross-spawn@^5.0.1:
shebang-command "^1.2.0"
which "^1.2.9"

[email protected]:
version "0.0.2"
resolved "https://registry.yarnpkg.com/cross-unzip/-/cross-unzip-0.0.2.tgz#5183bc47a09559befcf98cc4657964999359372f"
integrity sha1-UYO8R6CVWb78+YzEZXlkmZNZNy8=

crypto-browserify@^3.11.0:
version "3.12.0"
resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec"
Expand Down Expand Up @@ -4268,15 +4258,14 @@ electron-debug@^1.5.0:
electron-is-dev "^0.3.0"
electron-localshortcut "^3.0.0"

electron-devtools-installer@^2.2.4:
version "2.2.4"
resolved "https://registry.yarnpkg.com/electron-devtools-installer/-/electron-devtools-installer-2.2.4.tgz#261a50337e37121d338b966f07922eb4939a8763"
integrity sha512-b5kcM3hmUqn64+RUcHjjr8ZMpHS2WJ5YO0pnG9+P/RTdx46of/JrEjuciHWux6pE+On6ynWhHJF53j/EDJN0PA==
electron-devtools-installer@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/electron-devtools-installer/-/electron-devtools-installer-3.0.0.tgz#b5c439708746ed36f7b0220e18657567bca275d8"
integrity sha512-zll3w/8PvnPiGmL5tBtgSSoSjWnUljsOjJYsYYU12PKLljzWyfD6S75LKTZFn21VYxVbae2OwmjM5uFStLp6nQ==
dependencies:
"7zip" "0.0.6"
cross-unzip "0.0.2"
rimraf "^2.5.2"
semver "^5.3.0"
rimraf "^3.0.2"
semver "^7.2.1"
unzip-crx "^0.2.0"

electron-download@^3.0.1:
version "3.3.0"
Expand Down Expand Up @@ -7263,6 +7252,16 @@ jsprim@^1.2.2:
json-schema "0.2.3"
verror "1.10.0"

jszip@^3.1.0:
version "3.5.0"
resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.5.0.tgz#b4fd1f368245346658e781fec9675802489e15f6"
integrity sha512-WRtu7TPCmYePR1nazfrtuF216cIVon/3GWOvHS9QR5bIwSbnxtdpma6un3jyGGNhHsKCSzn5Ypk+EkDRvTGiFA==
dependencies:
lie "~3.3.0"
pako "~1.0.2"
readable-stream "~2.3.6"
set-immediate-shim "~1.0.1"

jszip@^3.1.5:
version "3.4.0"
resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.4.0.tgz#1a69421fa5f0bb9bc222a46bca88182fba075350"
Expand Down Expand Up @@ -10882,6 +10881,13 @@ [email protected]:
dependencies:
glob "^7.1.3"

rimraf@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==
dependencies:
glob "^7.1.3"

ripemd160@^2.0.0, ripemd160@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c"
Expand Down Expand Up @@ -11011,6 +11017,11 @@ semver@^6.0.0, semver@^6.2.0, semver@^6.3.0:
resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==

semver@^7.2.1:
version "7.3.2"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==

semver@~5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
Expand Down Expand Up @@ -12569,6 +12580,15 @@ unset-value@^1.0.0:
has-value "^0.3.1"
isobject "^3.0.0"

unzip-crx@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/unzip-crx/-/unzip-crx-0.2.0.tgz#4c0baa8bdac756256754beca7843c13d7b858c18"
integrity sha1-TAuqi9rHViVnVL7KeEPBPXuFjBg=
dependencies:
jszip "^3.1.0"
mkdirp "^0.5.1"
yaku "^0.16.6"

unzip-response@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97"
Expand Down Expand Up @@ -13442,6 +13462,11 @@ y18n@^3.2.1:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==

yaku@^0.16.6:
version "0.16.7"
resolved "https://registry.yarnpkg.com/yaku/-/yaku-0.16.7.tgz#1d195c78aa9b5bf8479c895b9504fd4f0847984e"
integrity sha1-HRlceKqbW/hHnIlblQT9TwhHmE4=

yallist@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
Expand Down

0 comments on commit 260dd63

Please sign in to comment.