Merge pull request #138 from TeselaGen/feature/ap/plugin-example-fixes

finishes plugin example

app/cli/src/handlers/manifest/generator.ts

@@ -2,7 +2,7 @@ import { uuid } from "@fsml/packages/utils/deps/mod.ts";
 import { set } from "@fsml/packages/utils/deps/lodash.ts";
 import { ManifestTypes } from "@fsml/cli/types/enums.ts";
 import {
-  createValueForType,
+  createTemplateForType,
   jsonToText,
   read,
 } from "@fsml/packages/utils/mod.ts";
@@ -40,10 +40,10 @@ const ManifestGenerator = (
   _opts?: ManifestGeneratorOpts,
 ) => {
   // Make an instance of the manifest if none is passed.
-  const manifest = _manifest || <TManifest> createValueForType(Manifest);
+  const manifest = _manifest || <TManifest> createTemplateForType(Manifest);
 
   function author(author: string): TProvenance {
-    const provenanceObject = <TProvenance> createValueForType(Provenance);
+    const provenanceObject = <TProvenance> createTemplateForType(Provenance);
     set(provenanceObject, "author", author);
     return provenanceObject;
   }
@@ -67,21 +67,23 @@ const ManifestGenerator = (
     const parserPlugin = await selectParser(filepath, parser);
 
     const SupplementalDataObject = <TSupplementalData> (
-      createValueForType(SupplementalData)
+      createTemplateForType(SupplementalData)
     );
 
     let dataObject: TFileData | TTabularData;
 
     // When no parser is available a 'TFileData' data object
     // will be generated.
     if (!parserPlugin) {
-      dataObject = <TFileData> createValueForType(FileData);
+      dataObject = <TFileData> createTemplateForType(FileData);
     } else {
       const result = await parserPlugin.run(filepath);
       dataObject = (result.data ||
-        createValueForType(TabularData)) as TTabularData;
-      if (!result.data && result.filepath) {
-        dataObject = JSON.parse(await read(result.filepath));
+        createTemplateForType(TabularData)) as TTabularData;
+      if (!result.data && result.file) {
+        dataObject = typeof result.file === "string"
+          ? JSON.parse(read(result.file))
+          : (new TextDecoder().decode(result.file)); // NOTE: we may need to specify that the encoding (s.a., UTF-8)
       }
     }
     SupplementalDataObject.data.push(dataObject);

app/cli/src/handlers/manifest/mod.ts

@@ -2,7 +2,7 @@ import { Arguments } from "@fsml/cli/deps/yargs.ts";
 import { jsonToText, remove, toStdOut } from "@fsml/packages/utils/mod.ts";
 import { generateManifest, packManifest, writeManifest } from "./utils.ts";
 
-/** CLI "manifest" commmand handlers **/
+/** CLI "manifest" command handlers **/
 
 async function create(args: Arguments) {
   const { type, parser, format, write, pack, author, filepattern } = args;
@@ -19,7 +19,7 @@ async function create(args: Arguments) {
 
   // Manifest will be sent to stdout when no write path is passed.
   if (!write) {
-    return await toStdOut(jsonToText({ content: manifest }));
+    return toStdOut(jsonToText({ content: manifest }));
   }
 
   const manifestFilepath = await writeManifest({ format, manifest });

app/cli/src/handlers/manifest/utils.ts

@@ -1,12 +1,12 @@
 import { path } from "@fsml/cli/deps/mod.ts";
-import { TypeCompiler } from "@fsml/packages/utils/deps/typebox.ts";
 import {
   expandGlobPaths,
   jsonToText,
   packFiles,
   remove,
   toFile,
   toStdOut,
+  validateType,
 } from "@fsml/packages/utils/mod.ts";
 import { DataFileFormats, ManifestTypes } from "@fsml/cli/types/enums.ts";
 import {
@@ -120,17 +120,15 @@ async function getDataFilepath(
 }
 
 function validateManifest(manifest: TManifest): boolean {
-  //@ts-ignore:next-line : This seems like an issue with typebox types.
-  const ManifestCompiler = TypeCompiler.Compile(Manifest);
-  const isValid = ManifestCompiler.Check(manifest);
-  const manifestErrors = [...ManifestCompiler.Errors(manifest)];
+  const { isValid, errors } = validateType(Manifest, manifest);
   if (!isValid) {
     toStdOut("Error in Manifest: \n");
-    manifestErrors.forEach((error) =>
-      // NOTE: Maybe extend it so that these errors can be written
-      // into a log file.
-      toStdOut(jsonToText({ format: "json", content: error }))
-    );
+    // TypeBox's TypeCompiler errors are quite verbosy and the escalate upwards the JSON tree.
+    // so if the error is located at a given leaf in the JSON tree, additional errors upwards the tree
+    // will be generated.
+
+    // For this reason, it might be best to throw the leaf error only, which is the first one.
+    console.info(errors[0]);
   }
   return isValid;
 }

app/cli/src/handlers/plugins/default-parser.ts

@@ -2,7 +2,7 @@
  * whatever interface we end up designing for data parsers. **/
 import { IParser, PluginTypes } from "@fsml/packages/plugins/types.ts";
 import { set } from "@fsml/packages/utils/deps/lodash.ts";
-import { createValueForType } from "@fsml/packages/utils/mod.ts";
+import { createTemplateForType } from "@fsml/packages/utils/mod.ts";
 import {
   TabularData,
   TTabularData,
@@ -24,12 +24,12 @@ import {
 const DefaultDataParser: IParser = {
   name: "defaultDataParser",
   type: PluginTypes.PARSER,
-  run: async (filepath: string) => {
+  run: async (filepath) => {
     console.info(`Parsing file '${filepath}'...`);
-    const tabularDataObject = <TTabularData> createValueForType(TabularData);
-    const columnObject = <TColumn> createValueForType(Column);
-    const kindObject = <TKind> createValueForType(Kind);
-    const classObject = <TClass> createValueForType(Class);
+    const tabularDataObject = <TTabularData> createTemplateForType(TabularData);
+    const columnObject = <TColumn> createTemplateForType(Column);
+    const kindObject = <TKind> createTemplateForType(Kind);
+    const classObject = <TClass> createTemplateForType(Class);
 
     set(kindObject, "class", classObject);
     set(columnObject, "kind", kindObject);
@@ -40,7 +40,7 @@ const DefaultDataParser: IParser = {
     return await Promise.resolve({ data: tabularDataObject });
   },
 
-  isApplicable: async (filepath: string) => {
+  isApplicable: async (filepath) => {
     console.info(`Checking if ${filepath} can be parsed...`);
     return await Promise.resolve(true);
   },

app/cli/src/handlers/plugins/handler/utils.ts

@@ -23,7 +23,9 @@ const defaultVersionResolver = async (
 
   const resp = await fetch(url);
   if (!resp.ok) {
-    throw new Error(`Error resolving module '${module.name}@${version}'`);
+    throw new Error(
+      `Error resolving module '${module.name}@${version} at ${url}'`,
+    );
   }
   if (resp.redirected) {
     const match = resp.url.match(versionRegex);

build-packages.ts

@@ -49,7 +49,8 @@ async function getPackageVersion(name: string): Promise<string> {
   } catch {
     /* module not published yet. */
   }
-  const newVersion = prompt(`Current version '${version}'. Select version: `);
+  const newVersion =
+    Deno.args[1] || prompt(`Current version '${version}'. Select version: `);
   return newVersion || version;
 }
 
@@ -66,11 +67,23 @@ async function build_npm(configs: BuildOptions) {
     entryPoints,
     outDir,
     package: _package,
-    shims = { deno: true },
+    shims = {
+      deno: true,
+      // Attempt to resolve this issue: https://github.com/denoland/deno/issues/16817
+      // custom: [
+      //   {
+      //     package: {
+      //       name: 'event-target-shim',
+      //       version: '6.0.2',
+      //     },
+      //     globalNames: ['internal'],
+      //   },
+      // ],
+    },
     ...rest
   } = configs;
 
-  console.log(`Emptying build dir '${outDir}'`);
+  console.info(`Emptying build dir '${outDir}'`);
   await emptyDir(outDir);
 
   await build({

package.json

@@ -11,7 +11,8 @@
     "build:plugins": "yarn bundle:plugins && deno run -A build-packages.ts plugins",
     "build:utils": "yarn bundle:utils && deno run -A build-packages.ts utils",
     "build:all": "yarn build:standard && yarn build:plugins && yarn build:utils",
-    "publish:standard": "(cd ./packages/standard/dist/npm && npm publish)"
+    "publish:standard": "(cd ./packages/standard/dist/npm && npm publish)",
+    "publish:utils": "(cd ./packages/utils/dist/npm && npm publish)"
   },
   "private": true,
   "dependencies": {},

packages/plugins/src/types.ts

@@ -13,7 +13,8 @@ export enum PluginTypes {
    */
   EXPORTER = "exporter",
   /**
-   * Any plugin used other purposes.
+   * Any plugin used for other purposes. Keep in mind these type of plugins
+   * will not necessarily work with all of the FSML SDK modules.
    */
   GENERIC = "generic",
 }
@@ -32,20 +33,19 @@ export interface IPlugin {
  * Parser interface that any Parser Plugin should extend.
  */
 export interface IParser extends IPlugin {
+  type: PluginTypes.PARSER;
   /**
-   * Receives filepath as input and returns
-   * a filepath as an output and/or along with the parsed data
-   * of type TTabularData
+   * Receives a file as input and returns an FSML manifest (optionally an output file).
    */
   run: (
-    filepath: string,
-  ) => Promise<
-    Partial<{
-      filepath: string;
-      data: TTabularData;
-    }>
-  >;
-  isApplicable: (filepath: string) => Promise<boolean>;
+    /* Input File, either as a filepath (string) or the file's data buffer stream (Uint8Array) */
+    file: string | Uint8Array,
+  ) => Promise<{
+    file?: string | Uint8Array;
+    data: TTabularData | TTabularData[];
+  }>;
+  /* Input File, either as a filepath (string) or the file's data buffer stream (Uint8Array) */
+  isApplicable: (file: string | Uint8Array) => Promise<boolean>;
 }
 
 /**
@@ -54,9 +54,9 @@ export interface IParser extends IPlugin {
 export interface IExporter extends IPlugin {
   /**
    * Receives an fsml manifest and returns a
-   * filepath as an output and/or along with
-   * a data object of some unknown type (e.g, json, yaml, etc.). */
+   * file and data object of some unknown type (e.g, json, yaml, etc.).
+   */
   run: (
     manifest: TManifest,
-  ) => Promise<Partial<{ filepath: string; data: unknown }>>;
+  ) => Promise<{ file: string | Uint8Array; data: unknown }>;
 }

packages/standard/src/mod.ts

@@ -1,13 +1,15 @@
 import { Manifest } from "./manifest/manifest.ts";
 import SupplementalData from "./manifest/data/supplemental-data.ts";
 import TabularData from "./manifest/data/tabular/mod.ts";
+import FileData from "./manifest/data/file-data.ts";
 import { Class, Column, Kind } from "./manifest/data/tabular/column/mod.ts";
 import Row from "./manifest/data/tabular/row.ts";
 import Value from "./manifest/data/tabular/value.ts";
 
 export {
   Class,
   Column,
+  FileData,
   Kind,
   Manifest,
   Row,

packages/utils/src/mod.ts

@@ -1,4 +1,4 @@
-import { Value } from "./deps/typebox.ts";
+import { TypeCompiler, Value } from "./deps/typebox.ts";
 import { conversion, fs, path, yaml } from "./deps/mod.ts";
 import compress from "./deps/compress.ts";
 
@@ -54,6 +54,30 @@ export function jsonToText(
   return text;
 }
 
+export function expandGlobPaths(filepattern: string) {
+  const filepaths = [];
+  for (const file of fs.expandGlobSync(filepattern)) {
+    filepaths.push(file.path);
+  }
+  return filepaths;
+}
+
+// TypeBox is not fully well supported for deno yet,
+// thus we have to use and ignore the 'any type here.
+// According to the docs 'createValueForType(type: TSchema)' should work.
+// deno-lint-ignore no-explicit-any
+export function createTemplateForType(type: any) {
+  return Value.Create(type);
+}
+
+// deno-lint-ignore no-explicit-any
+export function validateType(type: any, value: any) {
+  const ValueCompiler = TypeCompiler.Compile(type);
+  const isValid = ValueCompiler.Check(value);
+  const valueErrors = [...ValueCompiler.Errors(value)];
+  return { isValid, errors: valueErrors };
+}
+
 export async function packFiles(
   args: { pack: string; filepaths: string[]; write: string },
 ) {
@@ -75,22 +99,6 @@ export async function packFiles(
       break;
   }
 }
-
-export function expandGlobPaths(filepattern: string) {
-  const filepaths = [];
-  for (const file of fs.expandGlobSync(filepattern)) {
-    filepaths.push(file.path);
-  }
-  return filepaths;
-}
-
-// TypeBox is not fully well supported for deno yet,
-// thus we have to use and ignore the 'any type here.
-// According to the docs 'createValueForType(type: TSchema)' should work.
-// deno-lint-ignore no-explicit-any
-export function createValueForType(type: any) {
-  return Value.Create(type);
-}
 /**
  * Uses one of the available compressors to compress 'filepaths'
  * into the 'write' path.

website/docs/examples/microbyre-example.md

@@ -58,7 +58,7 @@ Now that we have the proper parser installed we can use it to parse one of the D
     $> fsml manifest create microbyreData.csv --parser microbyreParser --type data --author YOUR_NAME --format json --write mbManifest --pack tar
 ```
 
-Noted all the flags passed to the command? Well, we can set some of them as defaults so we don’t have to pass them in every time.
+Note all the flags passed to the command? Well, we can set some of them as defaults so we don’t have to pass them in every time.
 
 To do so we can use the “defaults” commands: