Skip to content

bpo-37774: use Py_LIKELY/Py_UNLIKELY for vectorcall #15144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jdemeyer wants to merge 3 commits into from
Closed

bpo-37774: use Py_LIKELY/Py_UNLIKELY for vectorcall #15144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4af535b
bpo-37774: add Py_LIKELY/Py_UNLIKELY macros
jdemeyer Aug 6, 2019
8aaaef8
bpo-37774: use Py_LIKELY/Py_UNLIKELY for vectorcall
jdemeyer Aug 6, 2019
7881bc9
bpo-37774: don't check GCC version before using __builtin_expect()
jdemeyer Aug 7, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Include/cpython/abstract.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ _PyVectorcall_Function(PyObject *callable)
{
assert(callable != NULL);
PyTypeObject *tp = Py_TYPE(callable);
if (!PyType_HasFeature(tp, _Py_TPFLAGS_HAVE_VECTORCALL)) {
if (!Py_LIKELY(PyType_HasFeature(tp, _Py_TPFLAGS_HAVE_VECTORCALL))) {
return NULL;
}
assert(PyCallable_Check(callable));
Expand Down
13 changes: 13 additions & 0 deletions Include/pymacro.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,17 @@

#define Py_UNREACHABLE() abort()


/* If we are using GCC, use __builtin_expect(). There is no need to check
* the GCC version since old versions that don't have __builtin_expect()
* can't compile CPython anyway */
#if defined(__GNUC__)
# define Py_UNLIKELY(value) __builtin_expect(!!(value), 0)
# define Py_LIKELY(value) __builtin_expect(!!(value), 1)
#else
# define Py_UNLIKELY(value) (!!(value))
# define Py_LIKELY(value) (!!(value))
#endif


#endif /* Py_PYMACRO_H */
46 changes: 18 additions & 28 deletions Objects/obmalloc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -710,16 +710,6 @@ PyObject_Free(void *ptr)
}


/* If we're using GCC, use __builtin_expect() to reduce overhead of
the valgrind checks */
#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)
# define UNLIKELY(value) __builtin_expect((value), 0)
# define LIKELY(value) __builtin_expect((value), 1)
#else
# define UNLIKELY(value) (value)
# define LIKELY(value) (value)
#endif

#ifdef WITH_PYMALLOC

#ifdef WITH_VALGRIND
Expand Down Expand Up @@ -1431,7 +1421,7 @@ address_in_range(void *p, poolp pool)
static void
pymalloc_pool_extend(poolp pool, uint size)
{
if (UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) {
if (Py_UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) {
/* There is room for another block. */
pool->freeblock = (block*)pool + pool->nextoffset;
pool->nextoffset += INDEX2SIZE(size);
Expand All @@ -1456,7 +1446,7 @@ allocate_from_new_pool(uint size)
/* There isn't a pool of the right size class immediately
* available: use a free pool.
*/
if (UNLIKELY(usable_arenas == NULL)) {
if (Py_UNLIKELY(usable_arenas == NULL)) {
/* No arena has a free pool: allocate a new arena. */
#ifdef WITH_MEMORY_LIMITS
if (narenas_currently_allocated >= MAX_ARENAS) {
Expand Down Expand Up @@ -1491,11 +1481,11 @@ allocate_from_new_pool(uint size)

/* Try to get a cached free pool. */
poolp pool = usable_arenas->freepools;
if (LIKELY(pool != NULL)) {
if (Py_LIKELY(pool != NULL)) {
/* Unlink from cached pools. */
usable_arenas->freepools = pool->nextpool;
usable_arenas->nfreepools--;
if (UNLIKELY(usable_arenas->nfreepools == 0)) {
if (Py_UNLIKELY(usable_arenas->nfreepools == 0)) {
/* Wholly allocated: remove. */
assert(usable_arenas->freepools == NULL);
assert(usable_arenas->nextarena == NULL ||
Expand Down Expand Up @@ -1590,26 +1580,26 @@ static inline void*
pymalloc_alloc(void *ctx, size_t nbytes)
{
#ifdef WITH_VALGRIND
if (UNLIKELY(running_on_valgrind == -1)) {
if (Py_UNLIKELY(running_on_valgrind == -1)) {
running_on_valgrind = RUNNING_ON_VALGRIND;
}
if (UNLIKELY(running_on_valgrind)) {
if (Py_UNLIKELY(running_on_valgrind)) {
return NULL;
}
#endif

if (UNLIKELY(nbytes == 0)) {
if (Py_UNLIKELY(nbytes == 0)) {
return NULL;
}
if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
if (Py_UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
return NULL;
}

uint size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;
poolp pool = usedpools[size + size];
block *bp;

if (LIKELY(pool != pool->nextpool)) {
if (Py_LIKELY(pool != pool->nextpool)) {
/*
* There is a used pool for this size class.
* Pick up the head block of its free list.
Expand All @@ -1618,7 +1608,7 @@ pymalloc_alloc(void *ctx, size_t nbytes)
bp = pool->freeblock;
assert(bp != NULL);

if (UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) {
if (Py_UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) {
// Reached the end of the free list, try to extend it.
pymalloc_pool_extend(pool, size);
}
Expand All @@ -1638,7 +1628,7 @@ static void *
_PyObject_Malloc(void *ctx, size_t nbytes)
{
void* ptr = pymalloc_alloc(ctx, nbytes);
if (LIKELY(ptr != NULL)) {
if (Py_LIKELY(ptr != NULL)) {
return ptr;
}

Expand All @@ -1657,7 +1647,7 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
size_t nbytes = nelem * elsize;

void* ptr = pymalloc_alloc(ctx, nbytes);
if (LIKELY(ptr != NULL)) {
if (Py_LIKELY(ptr != NULL)) {
memset(ptr, 0, nbytes);
return ptr;
}
Expand Down Expand Up @@ -1857,13 +1847,13 @@ pymalloc_free(void *ctx, void *p)
assert(p != NULL);

#ifdef WITH_VALGRIND
if (UNLIKELY(running_on_valgrind > 0)) {
if (Py_UNLIKELY(running_on_valgrind > 0)) {
return 0;
}
#endif

poolp pool = POOL_ADDR(p);
if (UNLIKELY(!address_in_range(p, pool))) {
if (Py_UNLIKELY(!address_in_range(p, pool))) {
return 0;
}
/* We allocated this address. */
Expand All @@ -1880,7 +1870,7 @@ pymalloc_free(void *ctx, void *p)
pool->freeblock = (block *)p;
pool->ref.count--;

if (UNLIKELY(lastfree == NULL)) {
if (Py_UNLIKELY(lastfree == NULL)) {
/* Pool was full, so doesn't currently live in any list:
* link it to the front of the appropriate usedpools[] list.
* This mimics LRU pool usage for new allocations and
Expand All @@ -1894,7 +1884,7 @@ pymalloc_free(void *ctx, void *p)
/* freeblock wasn't NULL, so the pool wasn't full,
* and the pool is in a usedpools[] list.
*/
if (LIKELY(pool->ref.count != 0)) {
if (Py_LIKELY(pool->ref.count != 0)) {
/* pool isn't empty: leave it in usedpools */
return 1;
}
Expand All @@ -1917,7 +1907,7 @@ _PyObject_Free(void *ctx, void *p)
return;
}

if (UNLIKELY(!pymalloc_free(ctx, p))) {
if (Py_UNLIKELY(!pymalloc_free(ctx, p))) {
/* pymalloc didn't allocate this address */
PyMem_RawFree(p);
raw_allocated_blocks--;
Expand Down Expand Up @@ -1945,7 +1935,7 @@ pymalloc_realloc(void *ctx, void **newptr_p, void *p, size_t nbytes)

#ifdef WITH_VALGRIND
/* Treat running_on_valgrind == -1 the same as 0 */
if (UNLIKELY(running_on_valgrind > 0)) {
if (Py_UNLIKELY(running_on_valgrind > 0)) {
return 0;
}
#endif
Expand Down
2 changes: 1 addition & 1 deletion Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4975,7 +4975,7 @@ call_function(PyThreadState *tstate, PyObject ***pp_stack, Py_ssize_t oparg, PyO
Py_ssize_t nargs = oparg - nkwargs;
PyObject **stack = (*pp_stack) - nargs - nkwargs;

if (tstate->use_tracing) {
if (Py_UNLIKELY(tstate->use_tracing)) {
x = trace_call_function(tstate, func, stack, nargs, kwnames);
}
else {
Expand Down