Skip to content

Commit 4743562

Browse files
fix(helpers): keep CGI + in query strings (#872)
* fix(helpers): keep CGI + in query strings normalize-url treats + as a space in valueless query keys and emits %20, which breaks CGI-style URLs like ?241+ful+HB1288+pdf. Co-authored-by: Cursor <[email protected]> * perf(helpers): skip CGI + restore when unused Most URLs have no `+` in the query; two indexOf checks avoid the restore path. Co-authored-by: Cursor <[email protected]> * refactor(helpers): use ternary for CGI + restore gate Co-authored-by: Cursor <[email protected]> * refactor(helpers): restore CGI + only when query has + Co-authored-by: Cursor <[email protected]> --------- Co-authored-by: Cursor <[email protected]>
1 parent 94d436e commit 4743562

2 files changed

Lines changed: 87 additions & 2 deletions

File tree

‎packages/metascraper-helpers/src/index.js‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,14 +176,66 @@ const protocol = url => {
176176
return ''
177177
}
178178

179-
const sanetizeUrl = (url, opts) =>
180-
_normalizeUrl(url, {
179+
/**
180+
* `normalize-url` treats `+` as a space in query keys that have no `=`
181+
* (CGI-style `?241+ful+HB1288+pdf`) and emits `%20`. Restore those pluses —
182+
* they are delimiters, not encoded spaces.
183+
*
184+
* @param {string} original
185+
* @param {string} normalized
186+
* @returns {string}
187+
*/
188+
const restoreCgiPlus = (original, normalized) => {
189+
const origMark = original.indexOf('?')
190+
const normMark = normalized.indexOf('?')
191+
if (origMark === -1 || normMark === -1) return normalized
192+
193+
const origHash = original.indexOf('#', origMark)
194+
const origSearch = original.slice(
195+
origMark + 1,
196+
origHash === -1 ? undefined : origHash
197+
)
198+
if (!origSearch.includes('+')) return normalized
199+
200+
const origByEncoded = new Map()
201+
for (const orig of origSearch.split('&')) {
202+
if (!orig.includes('=') && orig.includes('+')) {
203+
origByEncoded.set(orig.replaceAll('+', '%20'), orig)
204+
}
205+
}
206+
if (origByEncoded.size === 0) return normalized
207+
208+
const normHash = normalized.indexOf('#', normMark)
209+
const normSearch = normalized.slice(
210+
normMark + 1,
211+
normHash === -1 ? undefined : normHash
212+
)
213+
let changed = false
214+
const restored = normSearch.split('&').map(norm => {
215+
const orig = origByEncoded.get(norm)
216+
if (!orig) return norm
217+
changed = true
218+
return orig
219+
})
220+
if (!changed) return normalized
221+
222+
const hash = normHash === -1 ? '' : normalized.slice(normHash)
223+
return `${normalized.slice(0, normMark + 1)}${restored.join('&')}${hash}`
224+
}
225+
226+
const sanetizeUrl = (url, opts) => {
227+
const normalized = _normalizeUrl(url, {
181228
stripWWW: false,
182229
sortQueryParameters: false,
183230
removeSingleSlash: false,
184231
removeTrailingSlash: false,
185232
...opts
186233
})
234+
const queryStart = url.indexOf('?')
235+
return queryStart !== -1 && url.indexOf('+', queryStart) !== -1
236+
? restoreCgiPlus(url, normalized)
237+
: normalized
238+
}
187239

188240
const normalizeUrl = (baseUrl, relativePath, opts) => {
189241
try {

‎packages/metascraper-helpers/test/index.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,27 @@ test('.normalizeUrl', t => {
149149
undefined
150150
)
151151
t.is(normalizeUrl('https://www.example.com', 'javascript:void(0)'), undefined)
152+
t.is(
153+
normalizeUrl(
154+
'https://legacylis.virginia.gov/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
155+
),
156+
'https://legacylis.virginia.gov/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
157+
)
158+
t.is(
159+
normalizeUrl(
160+
'https://legacylis.virginia.gov/',
161+
'/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
162+
),
163+
'https://legacylis.virginia.gov/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
164+
)
165+
t.is(
166+
normalizeUrl('https://example.com/?q=hello+world'),
167+
'https://example.com/?q=hello+world'
168+
)
169+
t.is(
170+
normalizeUrl('https://example.com/?q=hello%20world'),
171+
'https://example.com/?q=hello%20world'
172+
)
152173
})
153174

154175
test('.author', t => {
@@ -215,6 +236,18 @@ test('.url', t => {
215236
),
216237
'http://cdn2.cloudpro.co.uk/sites/cloudprod7/files/4/29/handshake_0.jpg'
217238
)
239+
t.is(
240+
url(
241+
'https://legacylis.virginia.gov/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
242+
),
243+
'https://legacylis.virginia.gov/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
244+
)
245+
t.is(
246+
url('/cgi-bin/legp604.exe?241+ful+HB1288+pdf', {
247+
url: 'https://legacylis.virginia.gov/'
248+
}),
249+
'https://legacylis.virginia.gov/cgi-bin/legp604.exe?241+ful+HB1288+pdf'
250+
)
218251
})
219252

220253
test('.absoluteUrl', t => {

0 commit comments

Comments
 (0)