Skip to content

Commit 0f4abff

Browse files
fengmk2fengmk2
authored
feat: support nestedQuerystring as urllib v2 (#462)
closes #461
1 parent c7712a2 commit 0f4abff

7 files changed

Lines changed: 158 additions & 16 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
"formstream": "^1.1.1",
6969
"mime-types": "^2.1.35",
7070
"pump": "^3.0.0",
71+
"qs": "^6.11.2",
7172
"undici": "^5.22.1",
7273
"ylru": "^1.3.2"
7374
},
@@ -77,6 +78,7 @@
7778
"@types/mime-types": "^2.1.1",
7879
"@types/node": "^20.2.1",
7980
"@types/pump": "^1.1.1",
81+
"@types/qs": "^6.9.7",
8082
"@types/selfsigned": "^2.0.1",
8183
"@types/tar-stream": "^2.2.2",
8284
"@vitest/coverage-v8": "^0.32.0",

‎src/HttpClient.ts‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import { FormData as FormDataNode } from 'formdata-node';
2626
import { FormDataEncoder } from 'form-data-encoder';
2727
import createUserAgent from 'default-user-agent';
2828
import mime from 'mime-types';
29+
import qs from 'qs';
2930
import pump from 'pump';
3031
// Compatible with old style formstream
3132
import FormStream from 'formstream';
@@ -87,7 +88,7 @@ export type ClientOptions = {
8788
rejectUnauthorized?: boolean;
8889

8990
/**
90-
* sockePath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
91+
* socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
9192
*/
9293
socketPath?: string | null;
9394
},
@@ -244,14 +245,14 @@ export class HttpClient extends EventEmitter {
244245
// the response body and trailers have been received
245246
contentDownload: 0,
246247
};
247-
const orginalOpaque = args.opaque;
248+
const originalOpaque = args.opaque;
248249
// using opaque to diagnostics channel, binding request and socket
249250
const internalOpaque = {
250251
[symbols.kRequestId]: requestId,
251252
[symbols.kRequestStartTime]: requestStartTime,
252253
[symbols.kEnableRequestTiming]: !!args.timing,
253254
[symbols.kRequestTiming]: timing,
254-
[symbols.kRequestOrginalOpaque]: orginalOpaque,
255+
[symbols.kRequestOriginalOpaque]: originalOpaque,
255256
};
256257
const reqMeta = {
257258
requestId,
@@ -452,10 +453,17 @@ export class HttpClient extends EventEmitter {
452453
|| isReadable(args.data);
453454
if (isGETOrHEAD) {
454455
if (!isStringOrBufferOrReadable) {
455-
for (const field in args.data) {
456-
const fieldValue = args.data[field];
457-
if (fieldValue === undefined) continue;
458-
requestUrl.searchParams.append(field, fieldValue);
456+
if (args.nestedQuerystring) {
457+
const querystring = qs.stringify(args.data);
458+
// reset the requestUrl
459+
const href = requestUrl.href;
460+
requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
461+
} else {
462+
for (const field in args.data) {
463+
const fieldValue = args.data[field];
464+
if (fieldValue === undefined) continue;
465+
requestUrl.searchParams.append(field, fieldValue);
466+
}
459467
}
460468
}
461469
} else {
@@ -472,7 +480,11 @@ export class HttpClient extends EventEmitter {
472480
}
473481
} else {
474482
headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
475-
requestOptions.body = new URLSearchParams(args.data).toString();
483+
if (args.nestedQuerystring) {
484+
requestOptions.body = qs.stringify(args.data);
485+
} else {
486+
requestOptions.body = new URLSearchParams(args.data).toString();
487+
}
476488
}
477489
}
478490
}
@@ -582,7 +594,7 @@ export class HttpClient extends EventEmitter {
582594
this.#updateSocketInfo(socketInfo, internalOpaque);
583595

584596
const clientResponse: HttpClientResponse = {
585-
opaque: orginalOpaque,
597+
opaque: originalOpaque,
586598
data,
587599
status: res.status,
588600
statusCode: res.status,
@@ -637,7 +649,7 @@ export class HttpClient extends EventEmitter {
637649
return await this.#requestInternal(url, options, requestContext);
638650
}
639651
}
640-
err.opaque = orginalOpaque;
652+
err.opaque = originalOpaque;
641653
err.status = res.status;
642654
err.headers = res.headers;
643655
err.res = res;

‎src/Request.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ export type RequestOptions = {
4646
* Default is 'buffer'.
4747
*/
4848
dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
49+
/**
50+
* urllib default use URLSearchParams to stringify form data which don't support nested object,
51+
* will use qs instead of URLSearchParams to support nested object by set this option to true.
52+
*/
53+
nestedQuerystring?: boolean;
4954
/**
5055
* @deprecated
5156
* Only for d.ts keep compatible with urllib@2, don't use it anymore.

‎src/symbols.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ export default {
1212
kRequestStartTime: Symbol('request start time'),
1313
kEnableRequestTiming: Symbol('enable request timing or not'),
1414
kRequestTiming: Symbol('request timing'),
15-
kRequestOrginalOpaque: Symbol('request orginal opaque'),
15+
kRequestOriginalOpaque: Symbol('request original opaque'),
1616
};

‎test/diagnostics_channel.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ describe('diagnostics_channel.test.ts', () => {
4343
}
4444
}
4545
}
46-
const opaque = request[kHandler].opts.opaque[symbols.kRequestOrginalOpaque];
46+
const opaque = request[kHandler].opts.opaque[symbols.kRequestOriginalOpaque];
4747
if (opaque && name === 'undici:client:sendHeaders' && socket) {
4848
socket[kRequests]++;
4949
opaque.tracer.socket = {

‎test/fixtures/server.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { createReadStream } from 'node:fs';
66
import busboy from 'busboy';
77
import iconv from 'iconv-lite';
88
import selfsigned from 'selfsigned';
9+
import qs from 'qs';
910
import { readableToBytes, sleep } from '../utils';
1011

1112
const requestsPerSocket = Symbol('requestsPerSocket');
@@ -292,10 +293,20 @@ export async function startServer(options?: {
292293
}
293294

294295
if (req.headers['content-type']?.startsWith('application/x-www-form-urlencoded')) {
295-
const searchParams = new URLSearchParams(requestBytes.toString());
296-
requestBody = {};
297-
for (const [ field, value ] of searchParams.entries()) {
298-
requestBody[field] = value;
296+
const raw = requestBytes.toString();
297+
requestBody = {
298+
__raw__: raw,
299+
};
300+
if (req.headers['x-qs'] === 'true') {
301+
requestBody = {
302+
...qs.parse(raw),
303+
__raw__: raw,
304+
};
305+
} else {
306+
const searchParams = new URLSearchParams(raw);
307+
for (const [ field, value ] of searchParams.entries()) {
308+
requestBody[field] = value;
309+
}
299310
}
300311
} else if (req.headers['content-type']?.startsWith('application/json')) {
301312
requestBody = JSON.parse(requestBytes.toString());

‎test/options.data.test.ts‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { strict as assert } from 'node:assert';
22
import { createReadStream } from 'node:fs';
33
import { Readable } from 'node:stream';
4+
import qs from 'qs';
45
import { describe, it, beforeAll, afterAll } from 'vitest';
56
import urllib from '../src';
67
import { startServer } from './fixtures/server';
@@ -42,6 +43,82 @@ describe('options.data.test.ts', () => {
4243
assert.equal(url.searchParams.get('data'), '哈哈');
4344
});
4445

46+
it('should GET with data work on nestedQuerystring=true', async () => {
47+
const response = await urllib.request(_url, {
48+
method: 'GET',
49+
data: {
50+
sql: 'SELECT * from table',
51+
data: '哈哈',
52+
foo: {
53+
bar: 'bar value',
54+
array: [ 1, 2, 3 ],
55+
},
56+
},
57+
nestedQuerystring: true,
58+
dataType: 'json',
59+
headers: {
60+
'x-qs': 'true',
61+
},
62+
});
63+
assert.equal(response.status, 200);
64+
assert.equal(response.headers['content-type'], 'application/json');
65+
assert.equal(response.data.method, 'GET');
66+
assert(response.url.startsWith(_url));
67+
// console.log(response);
68+
assert(!response.redirected);
69+
assert.equal(response.data.url, '/?sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3');
70+
const query = qs.parse(response.data.url.substring(2));
71+
const url = new URL(response.data.href);
72+
assert.equal(url.searchParams.get('sql'), 'SELECT * from table');
73+
assert.equal(url.searchParams.get('data'), '哈哈');
74+
assert.equal(url.searchParams.get('foo[bar]'), 'bar value');
75+
assert.equal(url.searchParams.get('foo[array][0]'), '1');
76+
assert.equal(url.searchParams.get('foo[array][1]'), '2');
77+
assert.equal(url.searchParams.get('foo[array][2]'), '3');
78+
assert.equal(query.sql, 'SELECT * from table');
79+
assert.equal(query.data, '哈哈');
80+
assert.deepEqual(query.foo, { bar: 'bar value', array: [ '1', '2', '3' ] });
81+
});
82+
83+
it('should GET /ok?hello=1 with data work on nestedQuerystring=true', async () => {
84+
const response = await urllib.request(`${_url}ok?hello=1`, {
85+
method: 'GET',
86+
data: {
87+
sql: 'SELECT * from table',
88+
data: '哈哈',
89+
foo: {
90+
bar: 'bar value',
91+
array: [ 1, 2, 3 ],
92+
},
93+
},
94+
nestedQuerystring: true,
95+
dataType: 'json',
96+
headers: {
97+
'x-qs': 'true',
98+
},
99+
});
100+
assert.equal(response.status, 200);
101+
assert.equal(response.headers['content-type'], 'application/json');
102+
assert.equal(response.data.method, 'GET');
103+
assert(response.url.startsWith(_url));
104+
// console.log(response);
105+
assert(!response.redirected);
106+
assert.equal(response.data.url, '/ok?hello=1&sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3');
107+
const query = qs.parse(response.data.url.substring(4));
108+
const url = new URL(response.data.href);
109+
assert.equal(url.searchParams.get('hello'), '1');
110+
assert.equal(url.searchParams.get('sql'), 'SELECT * from table');
111+
assert.equal(url.searchParams.get('data'), '哈哈');
112+
assert.equal(url.searchParams.get('foo[bar]'), 'bar value');
113+
assert.equal(url.searchParams.get('foo[array][0]'), '1');
114+
assert.equal(url.searchParams.get('foo[array][1]'), '2');
115+
assert.equal(url.searchParams.get('foo[array][2]'), '3');
116+
assert.equal(query.hello, '1');
117+
assert.equal(query.sql, 'SELECT * from table');
118+
assert.equal(query.data, '哈哈');
119+
assert.deepEqual(query.foo, { bar: 'bar value', array: [ '1', '2', '3' ] });
120+
});
121+
45122
it('should HEAD with data and auto convert to query string', async () => {
46123
const response = await urllib.request(_url, {
47124
method: 'HEAD',
@@ -182,6 +259,7 @@ describe('options.data.test.ts', () => {
182259
assert.equal(response.data.headers['content-type'], 'application/x-www-form-urlencoded;charset=UTF-8');
183260
assert.equal(response.data.requestBody.sql, 'SELECT * from table');
184261
assert.equal(response.data.requestBody.data, '哈哈 PUT');
262+
assert.equal(response.data.requestBody.__raw__, 'sql=SELECT+*+from+table&data=%E5%93%88%E5%93%88+PUT');
185263
});
186264

187265
it('should PATCH with data and auto using application/x-www-form-urlencoded', async () => {
@@ -226,6 +304,40 @@ describe('options.data.test.ts', () => {
226304
assert.equal(response.data.requestBody.sql, 'SELECT * from table');
227305
assert.equal(response.data.requestBody.data, '哈哈 POST');
228306
assert.equal(response.data.requestBody.foo, '[object Object]');
307+
assert.equal(response.data.requestBody.__raw__, 'sql=SELECT+*+from+table&data=%E5%93%88%E5%93%88+POST&foo=%5Bobject+Object%5D');
308+
});
309+
310+
it('should POST with application/x-www-form-urlencoded work on nestedQuerystring=true', async () => {
311+
const response = await urllib.request(_url, {
312+
method: 'POST',
313+
data: {
314+
sql: 'SELECT * from table',
315+
data: '哈哈',
316+
foo: {
317+
bar: 'bar value',
318+
array: [ 1, 2, 3 ],
319+
},
320+
},
321+
nestedQuerystring: true,
322+
dataType: 'json',
323+
headers: {
324+
'x-qs': 'true',
325+
},
326+
});
327+
assert.equal(response.status, 200);
328+
assert.equal(response.headers['content-type'], 'application/json');
329+
assert.equal(response.data.method, 'POST');
330+
assert(response.url.startsWith(_url));
331+
assert(!response.redirected);
332+
// console.log(response.data);
333+
assert.equal(response.data.url, '/');
334+
assert.equal(response.data.headers['content-type'], 'application/x-www-form-urlencoded;charset=UTF-8');
335+
assert.equal(response.data.requestBody.sql, 'SELECT * from table');
336+
assert.equal(response.data.requestBody.data, '哈哈');
337+
assert(response.data.requestBody.foo, 'missing requestBody.foo');
338+
assert.equal(response.data.requestBody.foo.bar, 'bar value');
339+
assert.deepEqual(response.data.requestBody.foo.array, [ '1', '2', '3' ]);
340+
assert.equal(response.data.requestBody.__raw__, 'sql=SELECT%20%2A%20from%20table&data=%E5%93%88%E5%93%88&foo%5Bbar%5D=bar%20value&foo%5Barray%5D%5B0%5D=1&foo%5Barray%5D%5B1%5D=2&foo%5Barray%5D%5B2%5D=3');
229341
});
230342

231343
it('should PUT with data and contentType = json', async () => {

0 commit comments

Comments
 (0)