Skip to content

Commit 03768c4

Browse files
sweeneydesweeneyde
authored
bpo-45885: Specialize COMPARE_OP (GH-29734)
* Add COMPARE_OP_ADAPTIVE adaptive instruction. * Add COMPARE_OP_FLOAT_JUMP, COMPARE_OP_INT_JUMP and COMPARE_OP_STR_JUMP specialized instructions. * Introduce and use _PyUnicode_Equal
1 parent 99fcf15 commit 03768c4

File tree

9 files changed

+289
-60
lines changed

9 files changed

+289
-60
lines changed

‎Include/cpython/unicodeobject.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1016,6 +1016,9 @@ PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
10161016
and where the hash values are equal (i.e. a very probable match) */
10171017
PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *);
10181018

1019+
/* Equality check. Returns -1 on failure. */
1020+
PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *);
1021+
10191022
PyAPI_FUNC(int) _PyUnicode_WideCharString_Converter(PyObject *, void *);
10201023
PyAPI_FUNC(int) _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *);
10211024

‎Include/internal/pycore_code.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ typedef struct {
4242
uint16_t defaults_len;
4343
} _PyCallCache;
4444

45+
4546
/* Add specialized versions of entries to this union.
4647
*
4748
* Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
@@ -273,6 +274,7 @@ int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT
273274
int _Py_Specialize_CallFunction(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins);
274275
void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
275276
SpecializedCacheEntry *cache);
277+
void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
276278

277279
#define PRINT_SPECIALIZATION_STATS 0
278280
#define PRINT_SPECIALIZATION_STATS_DETAILED 0

‎Include/opcode.h‎

Lines changed: 41 additions & 37 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/opcode.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,10 @@ def jabs_op(name, op):
234234
"BINARY_OP_MULTIPLY_FLOAT",
235235
"BINARY_OP_SUBTRACT_INT",
236236
"BINARY_OP_SUBTRACT_FLOAT",
237+
"COMPARE_OP_ADAPTIVE",
238+
"COMPARE_OP_FLOAT_JUMP",
239+
"COMPARE_OP_INT_JUMP",
240+
"COMPARE_OP_STR_JUMP",
237241
"BINARY_SUBSCR_ADAPTIVE",
238242
"BINARY_SUBSCR_GETITEM",
239243
"BINARY_SUBSCR_LIST_INT",

‎Misc/NEWS.d/next/Core and Builtins/2021-11-23-21-01-56.bpo-45885.3IxeCX.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Specialized the ``COMPARE_OP`` opcode using the PEP 659 machinery.

‎Objects/unicodeobject.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11168,6 +11168,20 @@ unicode_compare_eq(PyObject *str1, PyObject *str2)
1116811168
return (cmp == 0);
1116911169
}
1117011170

11171+
int
11172+
_PyUnicode_Equal(PyObject *str1, PyObject *str2)
11173+
{
11174+
assert(PyUnicode_CheckExact(str1));
11175+
assert(PyUnicode_CheckExact(str2));
11176+
if (str1 == str2) {
11177+
return 1;
11178+
}
11179+
if (PyUnicode_READY(str1) || PyUnicode_READY(str2)) {
11180+
return -1;
11181+
}
11182+
return unicode_compare_eq(str1, str2);
11183+
}
11184+
1117111185

1117211186
int
1117311187
PyUnicode_Compare(PyObject *left, PyObject *right)

‎Python/ceval.c‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3778,6 +3778,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
37783778
}
37793779

37803780
TARGET(COMPARE_OP) {
3781+
PREDICTED(COMPARE_OP);
3782+
STAT_INC(COMPARE_OP, unquickened);
37813783
assert(oparg <= Py_GE);
37823784
PyObject *right = POP();
37833785
PyObject *left = TOP();
@@ -3792,6 +3794,125 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
37923794
DISPATCH();
37933795
}
37943796

3797+
TARGET(COMPARE_OP_ADAPTIVE) {
3798+
assert(cframe.use_tracing == 0);
3799+
SpecializedCacheEntry *cache = GET_CACHE();
3800+
if (cache->adaptive.counter == 0) {
3801+
PyObject *right = TOP();
3802+
PyObject *left = SECOND();
3803+
next_instr--;
3804+
_Py_Specialize_CompareOp(left, right, next_instr, cache);
3805+
DISPATCH();
3806+
}
3807+
else {
3808+
STAT_INC(COMPARE_OP, deferred);
3809+
cache->adaptive.counter--;
3810+
oparg = cache->adaptive.original_oparg;
3811+
STAT_DEC(COMPARE_OP, unquickened);
3812+
JUMP_TO_INSTRUCTION(COMPARE_OP);
3813+
}
3814+
}
3815+
3816+
TARGET(COMPARE_OP_FLOAT_JUMP) {
3817+
assert(cframe.use_tracing == 0);
3818+
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
3819+
SpecializedCacheEntry *caches = GET_CACHE();
3820+
int when_to_jump_mask = caches[0].adaptive.index;
3821+
PyObject *right = TOP();
3822+
PyObject *left = SECOND();
3823+
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
3824+
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
3825+
double dleft = PyFloat_AS_DOUBLE(left);
3826+
double dright = PyFloat_AS_DOUBLE(right);
3827+
int sign = (dleft > dright) - (dleft < dright);
3828+
DEOPT_IF(isnan(dleft), COMPARE_OP);
3829+
DEOPT_IF(isnan(dright), COMPARE_OP);
3830+
STAT_INC(COMPARE_OP, hit);
3831+
NEXTOPARG();
3832+
STACK_SHRINK(2);
3833+
Py_DECREF(left);
3834+
Py_DECREF(right);
3835+
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
3836+
int jump = (1 << (sign + 1)) & when_to_jump_mask;
3837+
if (!jump) {
3838+
next_instr++;
3839+
NOTRACE_DISPATCH();
3840+
}
3841+
else {
3842+
JUMPTO(oparg);
3843+
CHECK_EVAL_BREAKER();
3844+
NOTRACE_DISPATCH();
3845+
}
3846+
}
3847+
3848+
TARGET(COMPARE_OP_INT_JUMP) {
3849+
assert(cframe.use_tracing == 0);
3850+
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
3851+
SpecializedCacheEntry *caches = GET_CACHE();
3852+
int when_to_jump_mask = caches[0].adaptive.index;
3853+
PyObject *right = TOP();
3854+
PyObject *left = SECOND();
3855+
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
3856+
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
3857+
DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
3858+
DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
3859+
STAT_INC(COMPARE_OP, hit);
3860+
assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
3861+
Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
3862+
Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
3863+
int sign = (ileft > iright) - (ileft < iright);
3864+
NEXTOPARG();
3865+
STACK_SHRINK(2);
3866+
Py_DECREF(left);
3867+
Py_DECREF(right);
3868+
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
3869+
int jump = (1 << (sign + 1)) & when_to_jump_mask;
3870+
if (!jump) {
3871+
next_instr++;
3872+
NOTRACE_DISPATCH();
3873+
}
3874+
else {
3875+
JUMPTO(oparg);
3876+
CHECK_EVAL_BREAKER();
3877+
NOTRACE_DISPATCH();
3878+
}
3879+
}
3880+
3881+
TARGET(COMPARE_OP_STR_JUMP) {
3882+
assert(cframe.use_tracing == 0);
3883+
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
3884+
SpecializedCacheEntry *caches = GET_CACHE();
3885+
int invert = caches[0].adaptive.index;
3886+
PyObject *right = TOP();
3887+
PyObject *left = SECOND();
3888+
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
3889+
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
3890+
STAT_INC(COMPARE_OP, hit);
3891+
int res = _PyUnicode_Equal(left, right);
3892+
if (res < 0) {
3893+
goto error;
3894+
}
3895+
assert(caches[0].adaptive.original_oparg == Py_EQ ||
3896+
caches[0].adaptive.original_oparg == Py_NE);
3897+
NEXTOPARG();
3898+
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
3899+
STACK_SHRINK(2);
3900+
Py_DECREF(left);
3901+
Py_DECREF(right);
3902+
assert(res == 0 || res == 1);
3903+
assert(invert == 0 || invert == 1);
3904+
int jump = res ^ invert;
3905+
if (!jump) {
3906+
next_instr++;
3907+
NOTRACE_DISPATCH();
3908+
}
3909+
else {
3910+
JUMPTO(oparg);
3911+
CHECK_EVAL_BREAKER();
3912+
NOTRACE_DISPATCH();
3913+
}
3914+
}
3915+
37953916
TARGET(IS_OP) {
37963917
PyObject *right = POP();
37973918
PyObject *left = TOP();
@@ -5083,6 +5204,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
50835204
MISS_WITH_CACHE(LOAD_METHOD)
50845205
MISS_WITH_CACHE(CALL_FUNCTION)
50855206
MISS_WITH_CACHE(BINARY_OP)
5207+
MISS_WITH_CACHE(COMPARE_OP)
50865208
MISS_WITH_CACHE(BINARY_SUBSCR)
50875209
MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
50885210

‎Python/opcode_targets.h‎

Lines changed: 23 additions & 23 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)