Skip to content

Commit 9079efc

Browse files
Fix RuntimeError on ~2% of two-byte inputs (#368)
1 parent ea7e547 commit 9079efc

3 files changed

Lines changed: 51 additions & 1 deletion

File tree

‎src/chardet/pipeline/orchestrator.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -725,7 +725,8 @@ def _run_pipeline_core( # noqa: PLR0913
725725
results = _score_structural_candidates(
726726
data, structural_scores, valid_candidates, ctx
727727
)
728-
return _postprocess_results(data, results)
728+
if results:
729+
return _postprocess_results(data, results)
729730

730731
# Stage 3: Statistical scoring for all remaining candidates.
731732
# Bigram models converge quickly and don't benefit from scanning

‎tests/test_github_issues.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,39 @@ def test_issue_67_no_crash(self) -> None:
445445
assert isinstance(result, dict)
446446
assert "encoding" in result
447447

448+
def test_issue_367_short_two_byte_runtime_error(self) -> None:
449+
r"""Issue #367: b'\xf9\x92' raised RuntimeError.
450+
451+
Two-byte sequences that are valid multi-byte (e.g. cp932) score
452+
above the structural confidence threshold, but statistical scoring
453+
returns no results due to insufficient data. The pipeline must
454+
fall through to the fallback instead of returning an empty list.
455+
"""
456+
# The exact input from the bug report
457+
result = chardet.detect(b"\xf9\x92")
458+
assert isinstance(result, dict)
459+
assert "encoding" in result
460+
assert result["encoding"] is not None
461+
462+
def test_issue_367_additional_two_byte_sequences(self) -> None:
463+
"""Issue #367: verify other representative two-byte sequences don't crash.
464+
465+
Samples chosen from byte ranges that form valid cp932/johab multi-byte
466+
sequences — the same structural path that triggered the original bug.
467+
"""
468+
samples = [
469+
b"\x81\x40", # cp932 lead byte + valid trail
470+
b"\xf0\x80", # cp932 high lead byte
471+
b"\xe0\xa0", # cp932 lead byte
472+
b"\x84\x41", # johab lead byte
473+
b"\xd9\xfe", # johab high lead byte
474+
b"\xf9\x92", # original report
475+
]
476+
for data in samples:
477+
result = chardet.detect(data)
478+
assert isinstance(result, dict), f"Failed on {data!r}"
479+
assert "encoding" in result, f"Failed on {data!r}"
480+
448481

449482
# =========================================================================
450483
# NULL SEPARATOR ISSUES

‎tests/test_orchestrator.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,3 +372,19 @@ def empty_gate(data: bytes, valid_candidates: object, ctx: object) -> tuple[()]:
372372
data = bytes(range(0x80, 0x100)) * 2
373373
result = run_pipeline(data, EncodingEra.ALL)
374374
assert result[0].encoding is not None # fallback
375+
376+
377+
def test_fallback_when_structural_scores_high_but_statistical_empty():
378+
"""Fall through to fallback when structural scores high but statistical empty.
379+
380+
When structural scoring exceeds the threshold but statistical scoring
381+
returns no results (e.g. very short inputs), fall through to the
382+
no_match_encoding fallback instead of returning an empty list.
383+
384+
Regression test for GitHub issue #367.
385+
"""
386+
# b"\xf9\x92" is a valid cp932 multi-byte sequence that scores 1.0
387+
# structurally but yields no statistical bigram matches on 2 bytes.
388+
result = run_pipeline(b"\xf9\x92", EncodingEra.ALL)
389+
assert len(result) >= 1
390+
assert result[0].encoding is not None

0 commit comments

Comments
 (0)