Skip to content

Commit

Permalink
feat: File imports.
Browse files Browse the repository at this point in the history
  • Loading branch information
about-code committed Oct 21, 2021
1 parent 739ad35 commit 4447f72
Show file tree
Hide file tree
Showing 6 changed files with 244 additions and 36 deletions.
78 changes: 49 additions & 29 deletions conf/v5/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,7 @@
}
,"unified": {
"description": "Extended *unified* and *remark* configuration as described in https://github.com/unifiedjs/unified-engine/blob/main/doc/configure.md\nYou may want to provide such a configuration for loading *remark* plug-ins you've installed yourself. You likely require such plug-ins if your input files use third-party syntax which is not covered by the CommonMark specification. glossarify-md only supports CommonMark, GitHub Flavoured Markdown (GFM) and Footnotes by default. For additional remark plug-ins see https://github.com/remarkjs/awesome-remark\nNote that this configuration is not to be considered part of glossarify-md's own configuration interface! glossarify-md can not be held responsible for issues arising from loading additional plug-ins.\nIf you like to keep *unified* configuration separate use e.g. '{ \"unified\": { \"rcPath\": \"../unified.conf.json\"}} to load a unified configuration from an external file."
,"oneOf": [
{ "$ref": "#/$defs/unified"}
,{ "$ref": "#/$defs/UnifiedExternalConfig"}
]
,"$ref": "#/$defs/unified"
,"default": {}
}
,"dev": {
Expand Down Expand Up @@ -156,19 +153,6 @@
}
}
}
,"exportFile": {
"type": "object",
"properties": {
"file": {
"description": "The filename to write output to. Recommended extension is '.json'."
,"type": "string"
}
,"context": {
"description": "File name of a custom JSON-LD context document to embed. Should end with `.jsonld`. May also be JSON-LD document URL starting with `https://`."
,"type": "string"
}
}
}
,"glossaryFile": {
"type": "object"
,"properties": {
Expand All @@ -182,13 +166,26 @@
,"type": "array"
,"since": "6.0.0"
,"items": {
"$ref": "#/$defs/exportFile"
"$ref": "#/$defs/glossaryFileExports"
}
}
,"file": {
"description": "Name of the glossary file. Conventional default is *glossary.md*. You can use a glob pattern to enable cross-linking of headings across multiple files. Note that 'termHint' and 'title' will be ignored if 'file' is a glob pattern."
,"type": "string"
}
,"import": {
"description": "Import a JSON glossary (see 'export'). Generates a glossary markdown file from imported terms. Advanced: if the optional 'jsonld' library is installed glossarify-md will assume the JSON file to be a JSON-LD file. If it contains mappings of its custom attribute names onto well-known names from the W3C SKOS vocabulary then glossarify-md may understand the file even if it has a different structure than files exported by glossarify-md itself."
,"oneOf": [
{
"$ref": "#/$defs/glossaryFileImport"
}
,{
"type": "array",
"items": { "$ref": "#/$defs/glossaryFileImport" }
}
]
,"since": "6.0.0"
}
,"linkUris": {
"description": "Set this to true to hyperlink occurrences of a term to an 'authoritative' web glossary using a term's URI as lookup URL (default: false). May be used together with a glossary's 'uri' option. When 'linkUris' is 'true' glossarify-md uses the glossary markdown file as a source of link titles (tooltips) or for other internal processing, only, but won't generate links from documents to the markdown glossary, anymore, but from documents to an external web page."
,"type": "boolean"
Expand All @@ -210,6 +207,35 @@
,"since": "6.0.0"
}
}
,"not": {
"anyOf": [
{ "required": ["import", "export"] }
,{ "required": ["import", "exports"] }
,{ "required": ["export", "exports"] }
]
}
}
,"glossaryFileExports": {
"type": "object",
"properties": {
"file": {
"description": "A JSON file name to write exported terms to. Recommended file extension is '.json' or '.jsonld'"
,"type": "string"
}
,"context": {
"description": "File path or URL to a custom JSON-LD context document. JSON-LD contexts map terms from glossarify-md's export format onto terms of the well-known W3C SKOS vocabulary. If you want to import terms to another application supporting JSON-LD but not SKOS, then you can provide a custom JSON-LD context document with mappings of glossarify-md's terminology onto the one understood by the target application."
,"type": "string"
}
}
}
,"glossaryFileImport": {
"type": "object",
"properties": {
"file": {
"description": "The JSON file to import terms from."
,"type": "string"
}
}
}
,"indexFile": {
"type": "object"
Expand Down Expand Up @@ -375,22 +401,14 @@
}
}
}
,"UnifiedExternalConfig": {
,"unified": {
"type": "object"
,"title": "unified (external config)"
,"properties": {
"rcPath": {
"description": "Path to an external *unified* configuration file as documented under https://github.com/unifiedjs/unified-engine/blob/main/doc/configure.md. See description of *unified* property why you may want such a configuration."
,"type": "string"
}
}
,"required": ["rcPath"]
,"additionalProperties": false
}
,"unified": {
"type": "object"
,"properties": {
"plugins": {
,"plugins": {
"description": "Object or array with names of *unified* and *remark* plug-ins and plug-in settings as described in https://github.com/unifiedjs/unified-engine/blob/main/doc/configure.md\nNote that this configuration is not to be considered part of glossarify-md's own configuration interface!\nIf you like to keep *unified* configuration separate use 'rcPath' to load a unified configuration from an external file."
,"oneOf": [{
"type": "array"
Expand All @@ -412,7 +430,9 @@
}

}
,"additionalProperties": false
,"not": {
"required": ["plugins", "rcPath"]
}
}
,"dev": {
"type": "object"
Expand Down
175 changes: 175 additions & 0 deletions lib/importer.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
import { readFile } from "node:fs";
import path from "node:path";
import { VFile } from "vfile";

const SKOS = "http://www.w3.org/2004/02/skos/core#";
const SKOS_= "skos";
const DC = "http://purl.org/dc/terms/";
const DC_= "dc";
const LD_CONTEXT = {
[SKOS_]: SKOS
,[DC_]: DC
};
const SkosConceptSchemeType = {
uri: `${SKOS_}:ConceptScheme`
,title: `${DC_}:title`
};
const SkosConceptType = {
uri: `${SKOS_}:Concept`
,prefLabel: `${SKOS_}:prefLabel`
,altLabels: `${SKOS_}:altLabel`
,definition: `${SKOS_}:definition`
,abstract: `${DC_}:abstract`
};
/**
* @typedef {import("./model/context.js").Context } Context
*
* @param {Context} context
*/
export function importGlossaries(context) {
const {baseDir, glossaries} = context.conf;
return Promise.all(glossaries.map(g => {
const importConf = g.import;
if (! (importConf && g.file)) {
return Promise.resolve();
} else if (g.file.match(/[*|{}()]/g)) {
console.error(
`⚠ Config "glossaries": [{ "file": "${g.file}", "import": ... }]:`
+ "\nAttempts to write imported terms to multiple files using a glob."
+ "\nNot supported. Skipping import."
);
g.import = undefined;
return Promise.resolve();
}
return new Promise((resolve, reject) => {
const jsonFileName = importConf.file;
const jsonFile = path.resolve(baseDir, jsonFileName);
readFile(jsonFile, (err, jsonStr) => {
if (err) {
reject(err);
}
jsonParse(jsonStr).then(jsonData => {
try {
const vFile = new VFile({
path: g.file
,value: json2md(jsonData)
});
g.uri = jsonData.uri;
g.title = jsonData.title;
g.vFile = vFile;
context.writeFiles.push(vFile);
resolve();
} catch (parseErr) {
console.error(`${jsonFileName}: ${err.message}`);
reject(parseErr);
}
});
});
});
})).then(() => context);
}


function json2md(imported) {
const terms = imported.terms || {};
const termsStr = Object
.keys(terms)
.reduce(getTermReducer(terms), "");

const glossaryStr = `# ${imported.title || "Glossary" }\n${termsStr}`;
return glossaryStr;
}

function getTermReducer(terms) {
return function (acc, uri) {
const { value = "", definition = "", aliases = [] } = terms[uri];
const attrs = {
uri: uri
,aliases: aliases.join(", ")
};
const termAttrs = ["<!--", JSON.stringify(attrs, null, 2), "-->"].join("");
if (value) {
const md =
`
## ${value}
${termAttrs}
${definition}
`;
return md;
} else {
return "";
}
};
}

/**
* Parses the given JSON string.
* Uses plain JSON.parse() by default to deserialize JSON import data.
* Without 'jsonld' available it expects the data to align with glossarify-md's
* import format.
*
* If installed uses 'jsonld' to map JSON object onto a graph and interpret the
* JavaScript object using SKOS vocabulary. This enables to read other SKOS
* JSON-LD data formats. Installing jsonld is supposed to be done by users who
* wanto to import other SKOS-JSON formats with LD-Mappings.
*/
function jsonParse(jsonStr) {
try {
const jsonData = JSON.parse(jsonStr);
const glossary = import("jsonld")
.then(module => module.default)
.then(jsonld => {
jsonld.documentLoader = (url) => {
console.error(new Error(`${url}:\nLoading external JSON-LD documents not implemented.\n`));
};
return jsonld;
})
.then(jsonld => jsonld.flatten(jsonData, LD_CONTEXT))
.then(flattened => flattened["@graph"])
.then(ldNodes => {
const conceptScheme = ldNodes.find(ldNode => ldNode["@type"] === SkosConceptSchemeType.uri);
return {
// Map skos:ConceptScheme onto our 'Glossary' import format type
type: "Glossary"
,uri: ldValue(conceptScheme["@id"])
,title: ldValue(conceptScheme[SkosConceptSchemeType.title])
,terms: ldNodes
.filter(ldNode => ldNode["@type"] === SkosConceptType.uri)
.map(conceptInstance => {
// Map skos:Concept onto our 'Term' import format type
let aliases = ldValue(conceptInstance[SkosConceptType.altLabels]) || [];
if (aliases && !Array.isArray(aliases)) {
aliases = [aliases];
}
return {
type: "Term"
,uri: ldValue(conceptInstance["@id"])
,value: ldValue(conceptInstance[SkosConceptType.prefLabel])
,definition: ldValue(conceptInstance[SkosConceptType.definition])
,aliases: aliases
};
})
};
});

return glossary.catch(err => {
// console.log(err);
return jsonData;
});
} catch (err) {
Promise.reject(err);
}
}

function ldValue(v) {
if (Array.isArray(v)) {
if (v.length >= 1) {
return v[0]["@value"];
}
} else if (v && typeof v === "object") {
return v["@value"];
} else {
return v;
}
}
2 changes: 2 additions & 0 deletions lib/main.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import GitHubSlugger from "github-slugger";
import proc from "node:process";
import { importGlossaries } from "./importer.js";
import { newContext } from "./model/context.js";
import {
readDocumentFiles,
Expand All @@ -17,6 +18,7 @@ import {
export function run(glossarifyMdConf) {
newContext(glossarifyMdConf)
.then(context => copyBaseDirToOutDir(context))
.then(context => importGlossaries(context))
.then(context => readGlossaries(context))
.then(context => readDocumentFiles(context))
.then(context => Promise
Expand Down
17 changes: 15 additions & 2 deletions lib/model/context.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ import { init as initCollator } from "../text/collator.js";
import { Glossary } from "./glossary.js";


/**
*
* @param {*} glossarifyMdConf
* @returns {Context}
*/
export function newContext(glossarifyMdConf) {
initCollator(glossarifyMdConf.i18n);
const context = new Context(glossarifyMdConf);
Expand Down Expand Up @@ -86,8 +91,11 @@ function unglobGlossaries(context) {
};

const promises = glossaries.map((glossConf, idx) => {
const globPattern = glossConf.file;
let globPattern = glossConf.file;
return new Promise((resolve, reject) => {
if (!globPattern) {
return resolve([]);
}
glob(globPattern, globOpts, (err, files) => {
if (err) {
reject(err);
Expand All @@ -98,6 +106,9 @@ function unglobGlossaries(context) {
const file = relativeFromTo(baseDir, files[i]);
result[i] = { ...glossConf, file, arrIdx: idx };
}
if (numFiles === 0 && glossConf.import) {
result.push({ ...glossConf, arrIdx: idx });
}
resolve(result);
});
});
Expand All @@ -121,7 +132,9 @@ function unglobGlossaries(context) {
})
.filter(conf => {
const filePath = conf.file;
if (keys[filePath]) {
if (! filePath) {
return true;
} else if (keys[filePath]) {
return false;
} else {
return keys[filePath] = true;
Expand Down
2 changes: 2 additions & 0 deletions lib/model/glossary.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ export class Glossary {
,...[data.export].filter(v => !!v).map(v => { return {"file": v }; })
];

this.import = data.import || undefined;

/** @type {boolean} */
this.linkUris = data.linkUris || false;

Expand Down
6 changes: 1 addition & 5 deletions lib/reader.js
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,7 @@ export function readGlossaries(context) {
.use(indexer, { context, indexes: [...termsIndexes] })
.use(exporter, context )
,cwd: baseDir
,files: toForwardSlash(
glossaries
.map(g => g.file)
.sort((f1, f2) => f1.localeCompare(f2, "en"))
)
,files: glossaries.map(g => g.vFile ? g.vFile : g.file)
,ignoreName: ".mdignore"
,ignorePatterns: [
toForwardSlash(path.relative(baseDir, outDir))
Expand Down

0 comments on commit 4447f72

Please sign in to comment.