feat: new ignore_last_delimiters option, solve #193

adaltas · Jan 18, 2021 · 4d70b33 · 4d70b33
1 parent 121add5
commit 4d70b33
Show file tree

Hide file tree

Showing 8 changed files with 155 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,7 @@ Please join and contribute:
 
 ## Trunk
 
+* feat: new ignore_last_delimiters option, solve #193
 * feat: generate browser compatible lib
 * refactor: rename raw to record
 * docs: comment about trimable chars

diff --git a/lib/browser/index.js b/lib/browser/index.js
@@ -6744,6 +6744,24 @@ class Parser extends Transform {
         throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`)
       }
     }
+    // Normalize options `ignore_last_delimiters`
+    if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
+      options.ignore_last_delimiters = false
+    }else if(typeof options.ignore_last_delimiters === 'number'){
+      options.ignore_last_delimiters = options.ignore_last_delimiters > 0
+    }else if(typeof options.ignore_last_delimiters !== 'boolean'){
+      throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
+        'Invalid option `ignore_last_delimiters`:',
+        'the value must be a boolean value or an integer,',
+        `got ${JSON.stringify(options.ignore_last_delimiters)}`
+      ], options)
+    }
+    if(options.ignore_last_delimiters === true && options.columns === false){
+      throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
+        'The option `ignore_last_delimiters`',
+        'requires the activation of the `columns` option'
+      ], options)
+    }
     // Normalize option `info`
     if(options.info === undefined || options.info === null || options.info === false){
       options.info = false
@@ -7533,7 +7551,10 @@ class Parser extends Transform {
     return numOfCharLeft < requiredLength
   }
   __isDelimiter(buf, pos, chr){
-    const {delimiter} = this.options
+    const {delimiter, ignore_last_delimiters} = this.options
+    if(ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1){
+      return 0
+    }
     loop1: for(let i = 0; i < delimiter.length; i++){
       const del = delimiter[i]
       if(del[0] === chr){

diff --git a/lib/browser/sync.js b/lib/browser/sync.js
@@ -6744,6 +6744,24 @@ class Parser extends Transform {
         throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`)
       }
     }
+    // Normalize options `ignore_last_delimiters`
+    if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
+      options.ignore_last_delimiters = false
+    }else if(typeof options.ignore_last_delimiters === 'number'){
+      options.ignore_last_delimiters = options.ignore_last_delimiters > 0
+    }else if(typeof options.ignore_last_delimiters !== 'boolean'){
+      throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
+        'Invalid option `ignore_last_delimiters`:',
+        'the value must be a boolean value or an integer,',
+        `got ${JSON.stringify(options.ignore_last_delimiters)}`
+      ], options)
+    }
+    if(options.ignore_last_delimiters === true && options.columns === false){
+      throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
+        'The option `ignore_last_delimiters`',
+        'requires the activation of the `columns` option'
+      ], options)
+    }
     // Normalize option `info`
     if(options.info === undefined || options.info === null || options.info === false){
       options.info = false
@@ -7533,7 +7551,10 @@ class Parser extends Transform {
     return numOfCharLeft < requiredLength
   }
   __isDelimiter(buf, pos, chr){
-    const {delimiter} = this.options
+    const {delimiter, ignore_last_delimiters} = this.options
+    if(ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1){
+      return 0
+    }
     loop1: for(let i = 0; i < delimiter.length; i++){
       const del = delimiter[i]
       if(del[0] === chr){

diff --git a/lib/es5/index.js b/lib/es5/index.js
@@ -270,6 +270,19 @@ var Parser = /*#__PURE__*/function (_Transform) {
         } else {
           throw new Error("Invalid Option: from_line must be an integer, got ".concat(JSON.stringify(opts.from_line)));
         }
+      } // Normalize options `ignore_last_delimiters`
+
+
+      if (options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null) {
+        options.ignore_last_delimiters = false;
+      } else if (typeof options.ignore_last_delimiters === 'number') {
+        options.ignore_last_delimiters = options.ignore_last_delimiters > 0;
+      } else if (typeof options.ignore_last_delimiters !== 'boolean') {
+        throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', ['Invalid option `ignore_last_delimiters`:', 'the value must be a boolean value or an integer,', "got ".concat(JSON.stringify(options.ignore_last_delimiters))], options);
+      }
+
+      if (options.ignore_last_delimiters === true && options.columns === false) {
+        throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', ['The option `ignore_last_delimiters`', 'requires the activation of the `columns` option'], options);
       } // Normalize option `info`
 
 
@@ -1205,7 +1218,13 @@ var Parser = /*#__PURE__*/function (_Transform) {
   }, {
     key: "__isDelimiter",
     value: function __isDelimiter(buf, pos, chr) {
-      var delimiter = this.options.delimiter;
+      var _this$options5 = this.options,
+          delimiter = _this$options5.delimiter,
+          ignore_last_delimiters = _this$options5.ignore_last_delimiters;
+
+      if (ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1) {
+        return 0;
+      }
 
       loop1: for (var i = 0; i < delimiter.length; i++) {
         var del = delimiter[i];

diff --git a/lib/index.d.ts b/lib/index.d.ts
@@ -112,6 +112,10 @@ declare namespace parse {
          */
         from_line?: number;
         fromLine?: number;
+        /**
+         * Don't interpret delimiters as such in the last field according to the number of fields calculated from the number of columns, the option require the presence of the `column` option.
+         */
+        ignore_last_delimiters?: boolean | number;
         /**
          * Generate two properties `info` and `record` where `info` is a snapshot of the info object at the time the record was created and `record` is the parsed array or object.
          */

diff --git a/lib/index.js b/lib/index.js
@@ -205,6 +205,24 @@ class Parser extends Transform {
         throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`)
       }
     }
+    // Normalize options `ignore_last_delimiters`
+    if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
+      options.ignore_last_delimiters = false
+    }else if(typeof options.ignore_last_delimiters === 'number'){
+      options.ignore_last_delimiters = options.ignore_last_delimiters > 0
+    }else if(typeof options.ignore_last_delimiters !== 'boolean'){
+      throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
+        'Invalid option `ignore_last_delimiters`:',
+        'the value must be a boolean value or an integer,',
+        `got ${JSON.stringify(options.ignore_last_delimiters)}`
+      ], options)
+    }
+    if(options.ignore_last_delimiters === true && options.columns === false){
+      throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
+        'The option `ignore_last_delimiters`',
+        'requires the activation of the `columns` option'
+      ], options)
+    }
     // Normalize option `info`
     if(options.info === undefined || options.info === null || options.info === false){
       options.info = false
@@ -994,7 +1012,10 @@ class Parser extends Transform {
     return numOfCharLeft < requiredLength
   }
   __isDelimiter(buf, pos, chr){
-    const {delimiter} = this.options
+    const {delimiter, ignore_last_delimiters} = this.options
+    if(ignore_last_delimiters && this.state.record.length === this.options.columns.length - 1){
+      return 0
+    }
     loop1: for(let i = 0; i < delimiter.length; i++){
       const del = delimiter[i]
       if(del[0] === chr){

diff --git a/test/api.types.ts b/test/api.types.ts
@@ -30,12 +30,13 @@ describe('API Types', () => {
       const keys: string[] = Object.keys(options)
       keys.sort().should.eql([
         'bom', 'cast', 'cast_date', 'columns', 'columns_duplicates_to_array',
-        'comment', 'delimiter', 'encoding', 'escape', 'from', 'from_line', 'info', 'ltrim',
-        'max_record_size', 'objname', 'on_record', 'quote', 'raw',
-        'record_delimiter', 'relax', 'relax_column_count',
-        'relax_column_count_less', 'relax_column_count_more', 'rtrim',
-        'skip_empty_lines', 'skip_lines_with_empty_values',
-        'skip_lines_with_error', 'to',  'to_line', 'trim'
+        'comment', 'delimiter', 'encoding', 'escape', 'from', 'from_line',
+        'ignore_last_delimiters', 'info', 'ltrim', 'max_record_size', 'objname',
+        'on_record', 'quote', 'raw', 'record_delimiter', 'relax',
+        'relax_column_count', 'relax_column_count_less',
+        'relax_column_count_more', 'rtrim', 'skip_empty_lines',
+        'skip_lines_with_empty_values', 'skip_lines_with_error', 'to',
+        'to_line', 'trim'
       ])
     })
 
@@ -187,6 +188,12 @@ describe('API Types', () => {
       options.fromLine = 10
     })
 
+    it('ignore_last_delimiters', () => {
+      const options: Options = {}
+      options.ignore_last_delimiters = true
+      options.ignore_last_delimiters = 1
+    })
+
     it('info', () => {
       const options: Options = {}
       options.info = true

diff --git a/test/option.ignore_last_delimiters.coffee b/test/option.ignore_last_delimiters.coffee
@@ -0,0 +1,51 @@
+parse = require '../lib'
+
+describe 'Option `ignore_last_delimiters`', ->
+
+  describe 'validation & normalization', ->
+
+    it 'default to `false`', ->
+      parse().options.ignore_last_delimiters.should.eql false
+
+    it 'boolean', ->
+      parse(ignore_last_delimiters: true, columns: true).options.ignore_last_delimiters.should.eql true
+      parse(ignore_last_delimiters: false, columns: true).options.ignore_last_delimiters.should.eql false
+
+    it 'integer', ->
+      parse(ignore_last_delimiters: 1, columns: true).options.ignore_last_delimiters.should.eql true
+      parse(ignore_last_delimiters: 0, columns: true).options.ignore_last_delimiters.should.eql false
+
+    it 'throw error with invalid type', ->
+      (->
+        parse(ignore_last_delimiters: 'invalid')
+      ).should.throw
+        code: 'CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS'
+        message: [
+          'Invalid option `ignore_last_delimiters`:'
+          'the value must be a boolean value or an integer,'
+          'got "invalid"'
+        ].join ' '
+
+    it 'requires columns', ->
+      (->
+        parse(ignore_last_delimiters: true)
+      ).should.throw
+        code: 'CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS'
+        message: [
+          'The option `ignore_last_delimiters`'
+          'requires the activation of the `columns` option'
+        ].join ' '
+
+  describe 'usage', ->
+
+    it 'dont interpret last delimiters', (next) ->
+      parse '''
+      a,b,c
+      1,2,3,4,5
+      11,22,33,44
+      ''', ignore_last_delimiters: true, columns: true, (err, data) ->
+        data.should.eql [
+          {a: '1', b: '2', c: '3,4,5'}
+          {a: '11', b: '22', c: '33,44'}
+        ] unless err
+        next err