Skip to content

Commit 8a5eb7d

Browse files
wdavidwwdavidw
committed
feat: wg stream api
1 parent 89a6cd8 commit 8a5eb7d

76 files changed

Lines changed: 41378 additions & 58176 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎demo/eslint/.eslintrc.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
module.exports = {
2+
env: {
3+
browser: false,
4+
commonjs: true,
5+
node: true,
6+
es2021: true,
7+
},
8+
// Adding airbnb-base throw an error
9+
extends: ['eslint:recommended', 'plugin:import/recommended', 'airbnb-base'],
10+
parserOptions: {
11+
ecmaVersion: 'latest',
12+
},
13+
rules: {
14+
// 'import/no-unresolved': [2, { commonjs: false }],
15+
},
16+
};

‎demo/eslint/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
!.eslintrc.js

‎demo/eslint/index.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
const { stringify } = require('csv-stringify/sync');
2+
3+
const output = stringify([['a', 'b', 'c']]);
4+
5+
console.log(output);

‎demo/eslint/package.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"name": "csv-demo-eslint",
3+
"version": "0.0.1",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"lint": "eslint ./",
8+
"test": "echo \"Not ready, check output of npm run lint\""
9+
},
10+
"license": "MIT",
11+
"dependencies": {
12+
"csv-stringify": "^6.0.5"
13+
},
14+
"devDependencies": {
15+
"eslint": "^8.11.0",
16+
"eslint-config-airbnb-base": "^15.0.0",
17+
"eslint-plugin-import": "^2.25.4"
18+
}
19+
}

‎packages/csv-generate/dist/cjs/index.cjs‎

Lines changed: 122 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,64 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau
1010
var stream__default = /*#__PURE__*/_interopDefaultLegacy(stream);
1111
var util__default = /*#__PURE__*/_interopDefaultLegacy(util);
1212

13-
const Generator = function(options = {}){
13+
const init_state = (options) => {
14+
// State
15+
return {
16+
start_time: options.duration ? Date.now() : null,
17+
fixed_size_buffer: '',
18+
count_written: 0,
19+
count_created: 0,
20+
};
21+
};
22+
23+
// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option.
24+
const random = function(options={}){
25+
if(options.seed){
26+
return options.seed = options.seed * Math.PI * 100 % 100 / 100;
27+
}else {
28+
return Math.random();
29+
}
30+
};
31+
32+
const types = {
33+
// Generate an ASCII value.
34+
ascii: function({options}){
35+
const column = [];
36+
const nb_chars = Math.ceil(random(options) * options.maxWordLength);
37+
for(let i=0; i<nb_chars; i++){
38+
const char = Math.floor(random(options) * 32);
39+
column.push(String.fromCharCode(char + (char < 16 ? 65 : 97 - 16)));
40+
}
41+
return column.join('');
42+
},
43+
// Generate an integer value.
44+
int: function({options}){
45+
return Math.floor(random(options) * Math.pow(2, 52));
46+
},
47+
// Generate an boolean value.
48+
bool: function({options}){
49+
return Math.floor(random(options) * 2);
50+
}
51+
};
52+
53+
const camelize = function(str){
54+
return str.replace(/_([a-z])/gi, function(_, match){
55+
return match.toUpperCase();
56+
});
57+
};
58+
59+
const normalize_options = (opts) => {
1460
// Convert Stream Readable options if underscored
15-
if(options.high_water_mark){
16-
options.highWaterMark = options.high_water_mark;
61+
if(opts.high_water_mark){
62+
opts.highWaterMark = opts.high_water_mark;
1763
}
18-
if(options.object_mode){
19-
options.objectMode = options.object_mode;
64+
if(opts.object_mode){
65+
opts.objectMode = opts.object_mode;
2066
}
21-
// Call parent constructor
22-
stream__default["default"].Readable.call(this, options);
2367
// Clone and camelize options
24-
this.options = {};
25-
for(const k in options){
26-
this.options[Generator.camelize(k)] = options[k];
68+
const options = {};
69+
for(const k in opts){
70+
options[camelize(k)] = opts[k];
2771
}
2872
// Normalize options
2973
const dft = {
@@ -41,153 +85,142 @@ const Generator = function(options = {}){
4185
sleep: 0,
4286
};
4387
for(const k in dft){
44-
if(this.options[k] === undefined){
45-
this.options[k] = dft[k];
88+
if(options[k] === undefined){
89+
options[k] = dft[k];
4690
}
4791
}
4892
// Default values
49-
if(this.options.eof === true){
50-
this.options.eof = this.options.rowDelimiter;
93+
if(options.eof === true){
94+
options.eof = options.rowDelimiter;
5195
}
52-
// State
53-
this._ = {
54-
start_time: this.options.duration ? Date.now() : null,
55-
fixed_size_buffer: '',
56-
count_written: 0,
57-
count_created: 0,
58-
};
59-
if(typeof this.options.columns === 'number'){
60-
this.options.columns = new Array(this.options.columns);
96+
if(typeof options.columns === 'number'){
97+
options.columns = new Array(options.columns);
6198
}
62-
const accepted_header_types = Object.keys(Generator).filter((t) => (!['super_', 'camelize'].includes(t)));
63-
for(let i = 0; i < this.options.columns.length; i++){
64-
const v = this.options.columns[i] || 'ascii';
99+
const accepted_header_types = Object.keys(types).filter((t) => (!['super_', 'camelize'].includes(t)));
100+
for(let i = 0; i < options.columns.length; i++){
101+
const v = options.columns[i] || 'ascii';
65102
if(typeof v === 'string'){
66103
if(!accepted_header_types.includes(v)){
67104
throw Error(`Invalid column type: got "${v}", default values are ${JSON.stringify(accepted_header_types)}`);
68105
}
69-
this.options.columns[i] = Generator[v];
106+
options.columns[i] = types[v];
70107
}
71108
}
72-
return this;
109+
return options;
73110
};
74-
util__default["default"].inherits(Generator, stream__default["default"].Readable);
75111

76-
// Generate a random number between 0 and 1 with 2 decimals. The function is idempotent if it detect the "seed" option.
77-
Generator.prototype.random = function(){
78-
if(this.options.seed){
79-
return this.options.seed = this.options.seed * Math.PI * 100 % 100 / 100;
80-
}else {
81-
return Math.random();
82-
}
83-
};
84-
// Stop the generation.
85-
Generator.prototype.end = function(){
86-
this.push(null);
87-
};
88-
// Put new data into the read queue.
89-
Generator.prototype._read = function(size){
112+
const read = (options, state, size, push, close) => {
90113
// Already started
91114
const data = [];
92-
let length = this._.fixed_size_buffer.length;
115+
let length = state.fixed_size_buffer.length;
93116
if(length !== 0){
94-
data.push(this._.fixed_size_buffer);
117+
data.push(state.fixed_size_buffer);
95118
}
96119
// eslint-disable-next-line
97120
while(true){
98121
// Time for some rest: flush first and stop later
99-
if((this._.count_created === this.options.length) || (this.options.end && Date.now() > this.options.end) || (this.options.duration && Date.now() > this._.start_time + this.options.duration)){
122+
if((state.count_created === options.length) || (options.end && Date.now() > options.end) || (options.duration && Date.now() > state.start_time + options.duration)){
100123
// Flush
101124
if(data.length){
102-
if(this.options.objectMode){
125+
if(options.objectMode){
103126
for(const record of data){
104-
this.__push(record);
127+
push(record);
105128
}
106129
}else {
107-
this.__push(data.join('') + (this.options.eof ? this.options.eof : ''));
130+
push(data.join('') + (options.eof ? options.eof : ''));
108131
}
109-
this._.end = true;
132+
state.end = true;
110133
}else {
111-
this.push(null);
134+
close();
112135
}
113136
return;
114137
}
115138
// Create the record
116139
let record = [];
117140
let recordLength;
118-
this.options.columns.forEach((fn) => {
119-
record.push(fn(this));
141+
options.columns.forEach((fn) => {
142+
const result = fn({options: options, state: state});
143+
const type = typeof result;
144+
if(result !== null && type !== 'string' && type !== 'number'){
145+
throw Error([
146+
'INVALID_VALUE:',
147+
'values returned by column function must be',
148+
'a string, a number or null,',
149+
`got ${JSON.stringify(result)}`
150+
].join(' '));
151+
}
152+
record.push(result);
120153
});
121154
// Obtain record length
122-
if(this.options.objectMode){
155+
if(options.objectMode){
123156
recordLength = 0;
124157
// recordLength is currently equal to the number of columns
125158
// This is wrong and shall equal to 1 record only
126-
for(const column of record)
159+
for(const column of record){
127160
recordLength += column.length;
161+
}
128162
}else {
129163
// Stringify the record
130-
record = (this._.count_created === 0 ? '' : this.options.rowDelimiter)+record.join(this.options.delimiter);
164+
record = (state.count_created === 0 ? '' : options.rowDelimiter)+record.join(options.delimiter);
131165
recordLength = record.length;
132166
}
133-
this._.count_created++;
167+
state.count_created++;
134168
if(length + recordLength > size){
135-
if(this.options.objectMode){
169+
if(options.objectMode){
136170
data.push(record);
137171
for(const record of data){
138-
this.__push(record);
172+
push(record);
139173
}
140174
}else {
141-
if(this.options.fixedSize){
142-
this._.fixed_size_buffer = record.substr(size - length);
175+
if(options.fixedSize){
176+
state.fixed_size_buffer = record.substr(size - length);
143177
data.push(record.substr(0, size - length));
144178
}else {
145179
data.push(record);
146180
}
147-
this.__push(data.join(''));
181+
push(data.join(''));
148182
}
149183
return;
150184
}
151185
length += recordLength;
152186
data.push(record);
153187
}
154188
};
189+
190+
const Generator = function(options = {}){
191+
this.options = normalize_options(options);
192+
// Call parent constructor
193+
stream__default["default"].Readable.call(this, this.options);
194+
this.state = init_state(this.options);
195+
return this;
196+
};
197+
util__default["default"].inherits(Generator, stream__default["default"].Readable);
198+
199+
// Stop the generation.
200+
Generator.prototype.end = function(){
201+
this.push(null);
202+
};
203+
// Put new data into the read queue.
204+
Generator.prototype._read = function(size){
205+
const self = this;
206+
read(this.options, this.state, size, function(chunk) {
207+
self.__push(chunk);
208+
}, function(){
209+
self.push(null);
210+
});
211+
};
155212
// Put new data into the read queue.
156213
Generator.prototype.__push = function(record){
214+
// console.log('push', record)
157215
const push = () => {
158-
this._.count_written++;
216+
this.state.count_written++;
159217
this.push(record);
160-
if(this._.end === true){
218+
if(this.state.end === true){
161219
return this.push(null);
162220
}
163221
};
164222
this.options.sleep > 0 ? setTimeout(push, this.options.sleep) : push();
165223
};
166-
// Generate an ASCII value.
167-
Generator.ascii = function(gen){
168-
// Column
169-
const column = [];
170-
const nb_chars = Math.ceil(gen.random() * gen.options.maxWordLength);
171-
for(let i=0; i<nb_chars; i++){
172-
const char = Math.floor(gen.random() * 32);
173-
column.push(String.fromCharCode(char + (char < 16 ? 65 : 97 - 16)));
174-
}
175-
return column.join('');
176-
};
177-
// Generate an integer value.
178-
Generator.int = function(gen){
179-
return Math.floor(gen.random() * Math.pow(2, 52));
180-
};
181-
// Generate an boolean value.
182-
Generator.bool = function(gen){
183-
return Math.floor(gen.random() * 2);
184-
};
185-
// Camelize option properties
186-
Generator.camelize = function(str){
187-
return str.replace(/_([a-z])/gi, function(_, match){
188-
return match.toUpperCase();
189-
});
190-
};
191224

192225
const generate = function(){
193226
let options;

0 commit comments

Comments
 (0)