Skip to content

Commit fd75e66

Browse files
moroinemoroine
authored
fix(csv-parse): encoding detection with bom (#350)
1 parent e8b4fc7 commit fd75e66

10 files changed

Lines changed: 37 additions & 19 deletions

File tree

‎packages/csv-parse/dist/cjs/index.cjs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -635,8 +635,8 @@ const transform = function(original_options = {}) {
635635
},
636636
// Central parser implementation
637637
parse: function(nextBuf, end, push, close){
638-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
639-
let {record_delimiter} = this.options;
638+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
639+
let {comment, escape, quote, record_delimiter, } = this.options;
640640
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
641641
let buf;
642642
if(previousBuf === undefined){
@@ -672,6 +672,8 @@ const transform = function(original_options = {}) {
672672
buf = buf.slice(bomLength);
673673
// Renormalize original options with the new encoding
674674
this.options = normalize_options({...this.original_options, encoding: encoding});
675+
// Options will re-evaluate the Buffer with the new encoding
676+
({comment, escape, quote } = this.options);
675677
break;
676678
}
677679
}

‎packages/csv-parse/dist/cjs/sync.cjs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -633,8 +633,8 @@ const transform = function(original_options = {}) {
633633
},
634634
// Central parser implementation
635635
parse: function(nextBuf, end, push, close){
636-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
637-
let {record_delimiter} = this.options;
636+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
637+
let {comment, escape, quote, record_delimiter, } = this.options;
638638
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
639639
let buf;
640640
if(previousBuf === undefined){
@@ -670,6 +670,8 @@ const transform = function(original_options = {}) {
670670
buf = buf.slice(bomLength);
671671
// Renormalize original options with the new encoding
672672
this.options = normalize_options({...this.original_options, encoding: encoding});
673+
// Options will re-evaluate the Buffer with the new encoding
674+
({comment, escape, quote } = this.options);
673675
break;
674676
}
675677
}

‎packages/csv-parse/dist/esm/index.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5691,8 +5691,8 @@ const transform = function(original_options = {}) {
56915691
},
56925692
// Central parser implementation
56935693
parse: function(nextBuf, end, push, close){
5694-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
5695-
let {record_delimiter} = this.options;
5694+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
5695+
let {comment, escape, quote, record_delimiter, } = this.options;
56965696
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
56975697
let buf;
56985698
if(previousBuf === undefined){
@@ -5728,6 +5728,8 @@ const transform = function(original_options = {}) {
57285728
buf = buf.slice(bomLength);
57295729
// Renormalize original options with the new encoding
57305730
this.options = normalize_options({...this.original_options, encoding: encoding});
5731+
// Options will re-evaluate the Buffer with the new encoding
5732+
({comment, escape, quote } = this.options);
57315733
break;
57325734
}
57335735
}

‎packages/csv-parse/dist/esm/sync.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2601,8 +2601,8 @@ const transform = function(original_options = {}) {
26012601
},
26022602
// Central parser implementation
26032603
parse: function(nextBuf, end, push, close){
2604-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
2605-
let {record_delimiter} = this.options;
2604+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
2605+
let {comment, escape, quote, record_delimiter, } = this.options;
26062606
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
26072607
let buf;
26082608
if(previousBuf === undefined){
@@ -2638,6 +2638,8 @@ const transform = function(original_options = {}) {
26382638
buf = buf.slice(bomLength);
26392639
// Renormalize original options with the new encoding
26402640
this.options = normalize_options({...this.original_options, encoding: encoding});
2641+
// Options will re-evaluate the Buffer with the new encoding
2642+
({comment, escape, quote } = this.options);
26412643
break;
26422644
}
26432645
}

‎packages/csv-parse/dist/iife/index.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5694,8 +5694,8 @@ var csv_parse = (function (exports) {
56945694
},
56955695
// Central parser implementation
56965696
parse: function(nextBuf, end, push, close){
5697-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
5698-
let {record_delimiter} = this.options;
5697+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
5698+
let {comment, escape, quote, record_delimiter, } = this.options;
56995699
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
57005700
let buf;
57015701
if(previousBuf === undefined){
@@ -5731,6 +5731,8 @@ var csv_parse = (function (exports) {
57315731
buf = buf.slice(bomLength);
57325732
// Renormalize original options with the new encoding
57335733
this.options = normalize_options({...this.original_options, encoding: encoding});
5734+
// Options will re-evaluate the Buffer with the new encoding
5735+
({comment, escape, quote } = this.options);
57345736
break;
57355737
}
57365738
}

‎packages/csv-parse/dist/iife/sync.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2604,8 +2604,8 @@ var csv_parse_sync = (function (exports) {
26042604
},
26052605
// Central parser implementation
26062606
parse: function(nextBuf, end, push, close){
2607-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
2608-
let {record_delimiter} = this.options;
2607+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
2608+
let {comment, escape, quote, record_delimiter, } = this.options;
26092609
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
26102610
let buf;
26112611
if(previousBuf === undefined){
@@ -2641,6 +2641,8 @@ var csv_parse_sync = (function (exports) {
26412641
buf = buf.slice(bomLength);
26422642
// Renormalize original options with the new encoding
26432643
this.options = normalize_options({...this.original_options, encoding: encoding});
2644+
// Options will re-evaluate the Buffer with the new encoding
2645+
({comment, escape, quote } = this.options);
26442646
break;
26452647
}
26462648
}

‎packages/csv-parse/dist/umd/index.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5697,8 +5697,8 @@
56975697
},
56985698
// Central parser implementation
56995699
parse: function(nextBuf, end, push, close){
5700-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
5701-
let {record_delimiter} = this.options;
5700+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
5701+
let {comment, escape, quote, record_delimiter, } = this.options;
57025702
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
57035703
let buf;
57045704
if(previousBuf === undefined){
@@ -5734,6 +5734,8 @@
57345734
buf = buf.slice(bomLength);
57355735
// Renormalize original options with the new encoding
57365736
this.options = normalize_options({...this.original_options, encoding: encoding});
5737+
// Options will re-evaluate the Buffer with the new encoding
5738+
({comment, escape, quote } = this.options);
57375739
break;
57385740
}
57395741
}

‎packages/csv-parse/dist/umd/sync.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2607,8 +2607,8 @@
26072607
},
26082608
// Central parser implementation
26092609
parse: function(nextBuf, end, push, close){
2610-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
2611-
let {record_delimiter} = this.options;
2610+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
2611+
let {comment, escape, quote, record_delimiter, } = this.options;
26122612
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
26132613
let buf;
26142614
if(previousBuf === undefined){
@@ -2644,6 +2644,8 @@
26442644
buf = buf.slice(bomLength);
26452645
// Renormalize original options with the new encoding
26462646
this.options = normalize_options({...this.original_options, encoding: encoding});
2647+
// Options will re-evaluate the Buffer with the new encoding
2648+
({comment, escape, quote } = this.options);
26472649
break;
26482650
}
26492651
}

‎packages/csv-parse/lib/api/index.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ const transform = function(original_options = {}) {
5555
},
5656
// Central parser implementation
5757
parse: function(nextBuf, end, push, close){
58-
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
59-
let {record_delimiter} = this.options;
58+
const {bom, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
59+
let {comment, escape, quote, record_delimiter, } = this.options;
6060
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
6161
let buf;
6262
if(previousBuf === undefined){
@@ -92,6 +92,8 @@ const transform = function(original_options = {}) {
9292
buf = buf.slice(bomLength);
9393
// Renormalize original options with the new encoding
9494
this.options = normalize_options({...this.original_options, encoding: encoding});
95+
// Options will re-evaluate the Buffer with the new encoding
96+
({comment, escape, quote } = this.options);
9597
break;
9698
}
9799
}

‎packages/csv-parse/test/option.encoding.coffee‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,6 @@ describe 'Option `encoding`', ->
7878
] unless err
7979
next err
8080
# parser.write Buffer.from Buffer.from([255, 254])
81-
parser.write Buffer.from "\ufeffa,b,c\n", 'utf16le'
81+
parser.write Buffer.from '\ufeffa,b,"c"\n', 'utf16le'
8282
parser.write Buffer.from 'd,e,f', 'utf16le'
8383
parser.end()

0 commit comments

Comments
 (0)