Skip to content

Commit 7d18e20

Browse files
vstinnervstinner
authored
Merge branch 'main' into strerror_r
2 parents 2cebc12 + 978a8be commit 7d18e20

21 files changed

Lines changed: 584 additions & 346 deletions

File tree

‎Doc/data/stable_abi.dat‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Doc/using/configure.rst‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1022,6 +1022,19 @@ Debug options
10221022

10231023
.. versionadded:: 3.6
10241024

1025+
.. option:: --with-hwaddress-sanitizer
1026+
1027+
Enable HWAddressSanitizer memory error detector, ``hwasan`` (default is no).
1028+
Note that on x86-64 this uses `page aliasing
1029+
<https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html#supported-architectures>`_,
1030+
which only tags heap allocations and is unsafe for programs that ``fork()``,
1031+
including much of the test suite.
1032+
See the `LLVM HWASan design documentation
1033+
<https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html>`_
1034+
for more information.
1035+
1036+
.. versionadded:: next
1037+
10251038
.. option:: --with-memory-sanitizer
10261039

10271040
Enable MemorySanitizer allocation error detector, ``msan`` (default is no).

‎Doc/whatsnew/3.16.rst‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1171,6 +1171,11 @@ Build changes
11711171

11721172
(Contributed by Stan Ulbrych in :gh:`139314`.)
11731173

1174+
* Add the :option:`--with-hwaddress-sanitizer` :program:`configure` option to
1175+
build with `HWAddressSanitizer <https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html>`_.
1176+
1177+
(Contributed by Stan Ulbrych, Florian Mayer and AnnaAr321 in :gh:`156049`.)
1178+
11741179

11751180
C API changes
11761181
=============
@@ -1181,6 +1186,10 @@ New features
11811186
* Add :c:func:`Py_HashBuffer` to the limited C API.
11821187
(Contributed by Victor Stinner in :gh:`148233`.)
11831188

1189+
* Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C
1190+
API.
1191+
(Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.)
1192+
11841193
Porting to Python 3.16
11851194
----------------------
11861195

‎Include/cpython/object.h‎

Lines changed: 0 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -326,87 +326,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *,
326326

327327
PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *);
328328

329-
/* Safely decref `dst` and set `dst` to `src`.
330-
*
331-
* As in case of Py_CLEAR "the obvious" code can be deadly:
332-
*
333-
* Py_DECREF(dst);
334-
* dst = src;
335-
*
336-
* The safe way is:
337-
*
338-
* Py_SETREF(dst, src);
339-
*
340-
* That arranges to set `dst` to `src` _before_ decref'ing, so that any code
341-
* triggered as a side-effect of `dst` getting torn down no longer believes
342-
* `dst` points to a valid object.
343-
*
344-
* Temporary variables are used to only evaluate macro arguments once and so
345-
* avoid the duplication of side effects. _Py_TYPEOF(), C++ auto, or memcpy()
346-
* is used to avoid a miscompilation caused by type punning. See Py_CLEAR()
347-
* comment for implementation details about type punning.
348-
*
349-
* The memcpy() implementation does not emit a compiler warning if 'src' has
350-
* not the same type than 'src': any pointer type is accepted for 'src'.
351-
*/
352-
#ifdef _Py_TYPEOF
353-
#define Py_SETREF(dst, src) \
354-
do { \
355-
_Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
356-
_Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
357-
*_tmp_dst_ptr = (src); \
358-
Py_DECREF(_tmp_old_dst); \
359-
} while (0)
360-
#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L)
361-
#define Py_SETREF(dst, src) \
362-
do { \
363-
auto _tmp_dst_ptr = &(dst); \
364-
auto _tmp_old_dst = (*_tmp_dst_ptr); \
365-
*_tmp_dst_ptr = (src); \
366-
Py_DECREF(_tmp_old_dst); \
367-
} while (0)
368-
#else
369-
#define Py_SETREF(dst, src) \
370-
do { \
371-
PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
372-
PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
373-
PyObject *_tmp_src = _PyObject_CAST(src); \
374-
memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
375-
Py_DECREF(_tmp_old_dst); \
376-
} while (0)
377-
#endif
378-
379-
/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
380-
* Py_DECREF().
381-
*/
382-
#ifdef _Py_TYPEOF
383-
#define Py_XSETREF(dst, src) \
384-
do { \
385-
_Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
386-
_Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
387-
*_tmp_dst_ptr = (src); \
388-
Py_XDECREF(_tmp_old_dst); \
389-
} while (0)
390-
#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L)
391-
#define Py_XSETREF(dst, src) \
392-
do { \
393-
auto _tmp_dst_ptr = &(dst); \
394-
auto _tmp_old_dst = (*_tmp_dst_ptr); \
395-
*_tmp_dst_ptr = (src); \
396-
Py_XDECREF(_tmp_old_dst); \
397-
} while (0)
398-
#else
399-
#define Py_XSETREF(dst, src) \
400-
do { \
401-
PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
402-
PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
403-
PyObject *_tmp_src = _PyObject_CAST(src); \
404-
memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
405-
Py_XDECREF(_tmp_old_dst); \
406-
} while (0)
407-
#endif
408-
409-
410329
/* Define a pair of assertion macros:
411330
_PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT().
412331

‎Include/pyport.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,12 @@ extern "C" {
567567
# define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
568568
# endif
569569
# endif
570+
# if __has_feature(hwaddress_sanitizer)
571+
# if !defined(_Py_ADDRESS_SANITIZER)
572+
# define _Py_ADDRESS_SANITIZER
573+
# define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("hwaddress")))
574+
# endif
575+
# endif
570576
# if __has_feature(thread_sanitizer)
571577
# if !defined(_Py_THREAD_SANITIZER)
572578
# define _Py_THREAD_SANITIZER
@@ -577,6 +583,9 @@ extern "C" {
577583
# if defined(__SANITIZE_ADDRESS__)
578584
# define _Py_ADDRESS_SANITIZER
579585
# define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
586+
# elif defined(__SANITIZE_HWADDRESS__)
587+
# define _Py_ADDRESS_SANITIZER
588+
# define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("hwaddress")))
580589
# endif
581590
# if defined(__SANITIZE_THREAD__)
582591
# define _Py_THREAD_SANITIZER

‎Include/refcount.h‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,87 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
516516
#endif
517517

518518

519+
/* Safely decref `dst` and set `dst` to `src`.
520+
*
521+
* As in case of Py_CLEAR "the obvious" code can be deadly:
522+
*
523+
* Py_DECREF(dst);
524+
* dst = src;
525+
*
526+
* The safe way is:
527+
*
528+
* Py_SETREF(dst, src);
529+
*
530+
* That arranges to set `dst` to `src` _before_ decref'ing, so that any code
531+
* triggered as a side-effect of `dst` getting torn down no longer believes
532+
* `dst` points to a valid object.
533+
*
534+
* Temporary variables are used to only evaluate macro arguments once and so
535+
* avoid the duplication of side effects. _Py_TYPEOF(), C++ auto, or memcpy()
536+
* is used to avoid a miscompilation caused by type punning. See Py_CLEAR()
537+
* comment for implementation details about type punning.
538+
*
539+
* The memcpy() implementation does not emit a compiler warning if 'src' has
540+
* not the same type than 'src': any pointer type is accepted for 'src'.
541+
*/
542+
#ifdef _Py_TYPEOF
543+
#define Py_SETREF(dst, src) \
544+
do { \
545+
_Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
546+
_Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
547+
*_tmp_dst_ptr = (src); \
548+
Py_DECREF(_tmp_old_dst); \
549+
} while (0)
550+
#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L)
551+
#define Py_SETREF(dst, src) \
552+
do { \
553+
auto _tmp_dst_ptr = &(dst); \
554+
auto _tmp_old_dst = (*_tmp_dst_ptr); \
555+
*_tmp_dst_ptr = (src); \
556+
Py_DECREF(_tmp_old_dst); \
557+
} while (0)
558+
#else
559+
#define Py_SETREF(dst, src) \
560+
do { \
561+
PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
562+
PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
563+
PyObject *_tmp_src = _PyObject_CAST(src); \
564+
memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
565+
Py_DECREF(_tmp_old_dst); \
566+
} while (0)
567+
#endif
568+
569+
/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
570+
* Py_DECREF().
571+
*/
572+
#ifdef _Py_TYPEOF
573+
#define Py_XSETREF(dst, src) \
574+
do { \
575+
_Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
576+
_Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
577+
*_tmp_dst_ptr = (src); \
578+
Py_XDECREF(_tmp_old_dst); \
579+
} while (0)
580+
#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L)
581+
#define Py_XSETREF(dst, src) \
582+
do { \
583+
auto _tmp_dst_ptr = &(dst); \
584+
auto _tmp_old_dst = (*_tmp_dst_ptr); \
585+
*_tmp_dst_ptr = (src); \
586+
Py_XDECREF(_tmp_old_dst); \
587+
} while (0)
588+
#else
589+
#define Py_XSETREF(dst, src) \
590+
do { \
591+
PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
592+
PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
593+
PyObject *_tmp_src = _PyObject_CAST(src); \
594+
memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
595+
Py_XDECREF(_tmp_old_dst); \
596+
} while (0)
597+
#endif
598+
599+
519600
/* Function to use in case the object pointer can be NULL: */
520601
static inline void Py_XINCREF(PyObject *op)
521602
{

‎Lib/sysconfig/__init__.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -437,9 +437,10 @@ def parse_config_h(fp, vars=None):
437437
if vars is None:
438438
vars = {}
439439
import re
440-
define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
441-
undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
442-
quoted_re = re.compile('^"(.*)"$')
440+
name_rx = '(?:[A-Z]|_Py_)[A-Za-z0-9_]+'
441+
define_rx = re.compile(fr"#define ({name_rx}) (.*)\n")
442+
undef_rx = re.compile(fr"/[*] #undef ({name_rx}) [*]/\n")
443+
quoted_re = re.compile(r'"(.*)"')
443444

444445
while True:
445446
line = fp.readline()
@@ -448,7 +449,7 @@ def parse_config_h(fp, vars=None):
448449
m = define_rx.match(line)
449450
if m:
450451
n, v = m.group(1, 2)
451-
if mq := quoted_re.match(v):
452+
if mq := quoted_re.fullmatch(v):
452453
v = mq.group(1)
453454
try:
454455
if n in _ALWAYS_STR:

‎Lib/test/libregrtest/utils.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,9 @@ def get_build_info():
397397

398398
# --with-address-sanitizer
399399
sanitizers = []
400-
if support.check_sanitizer(address=True):
400+
if support.check_sanitizer(hwaddress=True):
401+
sanitizers.append("HWASAN")
402+
elif support.check_sanitizer(address=True):
401403
sanitizers.append("ASAN")
402404
# --with-memory-sanitizer
403405
if support.check_sanitizer(memory=True):

‎Lib/test/support/__init__.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -448,11 +448,12 @@ def skip_if_buildbot(reason=None):
448448
isbuildbot = False
449449
return unittest.skipIf(isbuildbot, reason)
450450

451-
def check_sanitizer(*, address=False, memory=False, ub=False, thread=False,
452-
function=True):
451+
def check_sanitizer(*, address=False, hwaddress=False, memory=False, ub=False,
452+
thread=False, function=True):
453453
"""Returns True if Python is compiled with sanitizer support"""
454-
if not (address or memory or ub or thread):
455-
raise ValueError('At least one of address, memory, ub or thread must be True')
454+
if not (address or hwaddress or memory or ub or thread):
455+
raise ValueError('At least one of address, hwaddress, memory, ub or '
456+
'thread must be True')
456457

457458

458459
cflags = sysconfig.get_config_var('CFLAGS') or ''
@@ -461,9 +462,14 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False,
461462
'-fsanitize=memory' in cflags or
462463
'--with-memory-sanitizer' in config_args
463464
)
465+
hwaddress_sanitizer = (
466+
'-fsanitize=hwaddress' in cflags or
467+
'--with-hwaddress-sanitizer' in config_args
468+
)
464469
address_sanitizer = (
465470
'-fsanitize=address' in cflags or
466-
'--with-address-sanitizer' in config_args
471+
'--with-address-sanitizer' in config_args or
472+
hwaddress_sanitizer
467473
)
468474
ub_sanitizer = (
469475
'-fsanitize=undefined' in cflags or
@@ -479,6 +485,7 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False,
479485
return (
480486
(memory and memory_sanitizer) or
481487
(address and address_sanitizer) or
488+
(hwaddress and hwaddress_sanitizer) or
482489
(ub and ub_sanitizer) or
483490
(thread and thread_sanitizer) or
484491
(function and function_sanitizer)

‎Lib/test/test_capi/test_mem.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,5 +182,18 @@ class PyMemDefaultTests(PyMemDebugTests):
182182
PYTHONMALLOC = ''
183183

184184

185+
@requires_subprocess()
186+
@unittest.skipUnless(support.check_sanitizer(address=True),
187+
'need address sanitizer')
188+
class AddressSanitizerTests(unittest.TestCase):
189+
def test_buffer_overflow(self):
190+
with support.SuppressCrashReport():
191+
out = assert_python_failure(
192+
'-c', 'import _testcapi; _testcapi.pymem_buffer_overflow()',
193+
PYTHONMALLOC='malloc',
194+
)
195+
self.assertIn(b'AddressSanitizer', out.err)
196+
197+
185198
if __name__ == "__main__":
186199
unittest.main()

0 commit comments

Comments
 (0)