Skip to content

Commit 40d0f06

Browse files
committed
Triple-check hardening: perceptron TOCTOU squat, pathsec fd-leak, bcp47 entity-safe XML (CWE-59/377/400/776)
Follow-up from an independent adversarial re-check of the previous commit (two fresh agents attacking the fixes + re-sweeping the whole repo). Three real items surfaced; each was reproduced out-of-process, fixed, and added to the harness. - tag/perceptron.py -- the previous `os.path.islink(loc)` pre-check on the trained-model save dir was non-atomic (TOCTOU) and inspected only the final component, so the guessable shared-temp name could still be raced or reached via a symlinked parent. Replace it with `_open_private_model_dir`: create the leaf with `os.mkdir` (atomic) and re-open with `O_NOFOLLOW|O_DIRECTORY` (refusing a pre-planted/raced symlink), verify it is a real directory owned by the current user and not group-/world-writable, and write each model file relative to that pinned directory fd with O_NOFOLLOW (CWE-59/377/378). - pathsec.py -- `_hardened_open` called `os.fdopen` outside the fd-closing guard. A mode string that `os.open` accepts but `os.fdopen` rejects (e.g. an empty mode) leaked the descriptor; repeated calls could exhaust the fd table (DoS). Wrap the `os.fdopen` so the fd is closed on any failure. - corpus/reader/bcp47.py -- the only XML corpus reader still using raw `ElementTree.parse`; a malicious `bcp47` data package could ship a billion-laughs `common-subdivisions-en.xml` (reachable via `nltk.langnames.langname()`), expanding internal entities to exhaust memory (CWE-776/400). Route it through `nltk.xmlsec.parse`, which refuses entity declarations, like every other XML consumer in the tree. Extend test_path_traversal_security.py: fd-leak-on-bad-mode, world-writable save-dir refusal, and bcp47 entity-expansion rejection + benign-parse. The independent re-check also confirmed the write-side symlink escape is genuinely closed (fd-realpath re-validation) and cleared the rest of the tree (framenet, wordnet_app whitelist, downloader zip, external-tool wrappers) with reasons. Full unit suite: 1521 passed, 0 failed; touched-module doctests green; pre-commit clean.
1 parent 483c5fe commit 40d0f06

4 files changed

Lines changed: 145 additions & 28 deletions

File tree

‎nltk/corpus/reader/bcp47.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77

88
import re
99
from warnings import warn
10-
from xml.etree import ElementTree as et
1110

1211
from nltk.corpus.reader import CorpusReader
12+
from nltk.xmlsec import parse as safe_parse
1313

1414

1515
class BCP47CorpusReader(CorpusReader):
@@ -37,7 +37,7 @@ def __init__(self, root, fileids):
3737
self.db = self.data_dict(fp.read().split("%%\n"))
3838
with self.open("cldr/common-subdivisions-en.xml") as fp:
3939
self.subdiv = self.subdiv_dict(
40-
et.parse(fp).iterfind("localeDisplayNames/subdivisions/subdivision")
40+
safe_parse(fp).iterfind("localeDisplayNames/subdivisions/subdivision")
4141
)
4242
self.morphology()
4343

‎nltk/pathsec.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -734,7 +734,14 @@ def _hardened_open(raw_path, mode, context, required_root, **kwargs):
734734
except BaseException:
735735
os.close(fd)
736736
raise
737-
return os.fdopen(fd, mode, **kwargs)
737+
try:
738+
return os.fdopen(fd, mode, **kwargs)
739+
except BaseException:
740+
# os.open() accepts some mode strings that os.fdopen() rejects (e.g. an
741+
# empty or malformed mode); without this the descriptor would leak and
742+
# repeated calls could exhaust the fd table (DoS).
743+
os.close(fd)
744+
raise
738745

739746

740747
# Back-compat alias: the opener now handles write modes too, but external

‎nltk/tag/perceptron.py‎

Lines changed: 72 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,49 @@ def _authorize_private_dir(directory):
4848
pathsec._LAST_DATA_PATHS = None
4949

5050

51+
def _open_private_model_dir(loc):
52+
"""Create/reuse *loc* as a private directory and return a pinned O_NOFOLLOW
53+
directory descriptor, closing the shared-temp symlink squat / TOCTOU race.
54+
55+
The leaf is created with ``os.mkdir`` (atomic: it either creates the real
56+
directory or fails with ``FileExistsError``) and then re-opened with
57+
``O_NOFOLLOW|O_DIRECTORY`` -- so a symlink pre-planted or raced in at the
58+
guessable default name is refused (``ELOOP``/``ENOTDIR``) instead of being
59+
followed. The descriptor is rejected unless it is a real directory owned by
60+
the current user and not group-/world-writable (CWE-59/377/378).
61+
"""
62+
import errno
63+
64+
parent = os.path.dirname(loc) or "."
65+
os.makedirs(parent, exist_ok=True)
66+
try:
67+
os.mkdir(loc, 0o700)
68+
except FileExistsError:
69+
pass
70+
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
71+
try:
72+
fd = os.open(loc, flags)
73+
except OSError as e:
74+
if e.errno in (errno.ELOOP, errno.ENOTDIR):
75+
raise PermissionError(
76+
f"Refusing trained-model dir {loc!r}: not a real directory "
77+
"(symlink / TOCTOU squat) (CWE-59)"
78+
) from e
79+
raise
80+
try:
81+
st = os.fstat(fd)
82+
# POSIX only: on Windows st_uid is not meaningful and NTFS ACLs govern.
83+
if os.name == "posix" and (st.st_uid != os.getuid() or (st.st_mode & 0o022)):
84+
raise PermissionError(
85+
f"Refusing non-private trained-model dir {loc!r}: not owned by "
86+
"you or group-/world-writable (CWE-377/378)"
87+
)
88+
except BaseException:
89+
os.close(fd)
90+
raise
91+
return fd
92+
93+
5194
try:
5295
import numpy as np
5396
except ImportError:
@@ -289,31 +332,35 @@ def train(self, sentences, save_loc=None, nr_iter=5):
289332
def save_to_json(self, lang="xxx", loc=None):
290333
if not loc:
291334
loc = self.save_dir
292-
# Create the trained-model dir private to the current user (mode 0700).
293-
# The default TRAINED_TAGGER_PATH is the system temp dir, which is shared
294-
# and world-writable on Linux; a private dir cannot be tampered with by
295-
# another local user, and lets it be authorized for reading back below.
296-
#
297-
# The default location is a *guessable* name in a shared temp dir, so a
298-
# local attacker can pre-create it. Refuse a symlink there outright
299-
# (``makedirs(exist_ok=True)`` would silently accept and then follow it,
300-
# redirecting the model write outside the intended dir -- CWE-59/377).
301-
if os.path.islink(loc):
302-
raise PermissionError(
303-
f"Refusing to save trained model into a symlink: {loc!r} (CWE-59)"
304-
)
305-
os.makedirs(loc, mode=0o700, exist_ok=True)
306-
_authorize_private_dir(loc)
307-
308-
# Open each model file with O_NOFOLLOW (0600) so a symlink swapped in at
309-
# the individual file name is refused atomically rather than followed,
310-
# and the written model is never left group-/world-readable.
311-
def _no_follow_opener(path, flags):
312-
return os.open(path, flags | getattr(os, "O_NOFOLLOW", 0), 0o600)
313-
314-
for param, json_file in zip(self.encode_json_obj(), self.param_files(lang)):
315-
with open(path_join(loc, json_file), "w", opener=_no_follow_opener) as fout:
316-
json.dump(param, fout)
335+
# The default TRAINED_TAGGER_PATH is the system temp dir -- shared and
336+
# world-writable on Linux -- and the save dir is a *guessable* name in
337+
# it, so a local attacker can pre-plant or race a symlink at ``loc``. A
338+
# plain ``islink`` pre-check is non-atomic (TOCTOU) and misses it once
339+
# created; ``_open_private_model_dir`` instead creates/re-opens the leaf
340+
# atomically with O_NOFOLLOW|O_DIRECTORY and verifies it is a real,
341+
# user-owned, non-world-writable directory, returning a pinned fd
342+
# (CWE-59/377/378).
343+
dir_fd = _open_private_model_dir(loc)
344+
try:
345+
_authorize_private_dir(loc)
346+
347+
# Write each model file relative to the pinned directory fd (where
348+
# supported) with O_NOFOLLOW (0600), so neither the dir nor the file
349+
# can be redirected outside the verified directory after the check.
350+
use_dir_fd = os.open in os.supports_dir_fd
351+
352+
def _no_follow_opener(path, flags):
353+
extra = {"dir_fd": dir_fd} if use_dir_fd else {}
354+
return os.open(
355+
path, flags | getattr(os, "O_NOFOLLOW", 0), 0o600, **extra
356+
)
357+
358+
for param, json_file in zip(self.encode_json_obj(), self.param_files(lang)):
359+
target = json_file if use_dir_fd else path_join(loc, json_file)
360+
with open(target, "w", opener=_no_follow_opener) as fout:
361+
json.dump(param, fout)
362+
finally:
363+
os.close(dir_fd)
317364

318365
def load_from_json(self, lang="eng", loc=None):
319366
# Automatically find path to the tagger if location is not specified.

‎nltk/test/unit/test_path_traversal_security.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,21 @@ def test_hardlink_to_outside_inode_is_refused(self, monkeypatch):
256256
with pytest.raises(PermissionError):
257257
P.open(hard, "r", required_root=root).read()
258258

259+
def test_malformed_mode_does_not_leak_fd(self):
260+
# os.open() accepts some mode strings os.fdopen() rejects (e.g. ""). The
261+
# fdopen must be inside the fd-closing guard or repeated calls exhaust
262+
# the fd table (DoS).
263+
P, base, root, outside = self._roots()
264+
f = os.path.join(root, "x")
265+
with open(f, "w") as fh:
266+
fh.write("hi")
267+
before = len(os.listdir("/dev/fd"))
268+
for _ in range(64):
269+
with pytest.raises((ValueError, OSError)):
270+
P.open(f, "", required_root=root)
271+
after = len(os.listdir("/dev/fd"))
272+
assert after <= before + 2, f"fd leak: {before} -> {after}"
273+
259274

260275
# ==========================================================================
261276
# 3. Predictable shared-temp squat on the trained-model save (CWE-59/377)
@@ -296,3 +311,51 @@ def test_symlinked_save_location_is_refused(self):
296311
with pytest.raises(PermissionError):
297312
self._tagger().save_to_json(lang="eng", loc=squat)
298313
assert not os.listdir(victim), "write leaked through the symlink"
314+
315+
def test_world_writable_save_location_is_refused(self):
316+
# A pre-existing real dir at the guessable name that is world-writable
317+
# (an attacker could drop files in it / read the model back) is refused.
318+
base = tempfile.mkdtemp()
319+
loc = os.path.join(base, "shared")
320+
os.makedirs(loc)
321+
os.chmod(loc, 0o777)
322+
with pytest.raises(PermissionError):
323+
self._tagger().save_to_json(lang="eng", loc=loc)
324+
325+
326+
# ==========================================================================
327+
# 4. Entity-expansion (billion-laughs) in the bcp47 XML corpus reader (CWE-776)
328+
# ==========================================================================
329+
330+
331+
class TestBcp47EntitySafeParse:
332+
def test_entity_expansion_is_rejected(self):
333+
# bcp47 was the only XML corpus reader using raw ElementTree.parse; a
334+
# malicious `bcp47` package could ship a billion-laughs subdivisions XML.
335+
# It now goes through nltk.xmlsec, which refuses entity declarations.
336+
import io
337+
338+
from nltk.xmlsec import parse as safe_parse
339+
340+
lol = (
341+
'<?xml version="1.0"?><!DOCTYPE lolz ['
342+
'<!ENTITY a "AAAA"><!ENTITY b "&a;&a;&a;&a;&a;">]>'
343+
"<localeDisplayNames><subdivisions><subdivision>&b;"
344+
"</subdivision></subdivisions></localeDisplayNames>"
345+
)
346+
with pytest.raises(Exception):
347+
safe_parse(io.BytesIO(lol.encode()))
348+
349+
def test_benign_subdivisions_xml_still_parses(self):
350+
import io
351+
352+
from nltk.xmlsec import parse as safe_parse
353+
354+
benign = (
355+
"<localeDisplayNames><subdivisions>"
356+
'<subdivision type="usca">California</subdivision>'
357+
"</subdivisions></localeDisplayNames>"
358+
)
359+
tree = safe_parse(io.BytesIO(benign.encode()))
360+
subs = list(tree.iterfind("subdivisions/subdivision"))
361+
assert len(subs) == 1 and subs[0].text == "California"

0 commit comments

Comments
 (0)