Skip to content

Commit eef19a3

Browse files
committed
fix: update json path parser, do not apply partial updates in delete by key, add tests
1 parent 3f0cfaa commit eef19a3

6 files changed

Lines changed: 90 additions & 34 deletions

File tree

‎qdrant_client/local/json_path_parser.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
from pydantic import BaseModel
44

55

6+
U64_MAX = 2**64 - 1
7+
8+
69
class JsonPathItemType(str, Enum):
710
KEY = "key"
811
INDEX = "index"
@@ -140,11 +143,15 @@ def _match_brackets(path: str) -> tuple[JsonPathItem | None, str]:
140143
path[right_bracket_pos + 1 :],
141144
)
142145

143-
try:
144-
index = int(path[left_bracket_pos + 1 : right_bracket_pos])
145-
return (
146-
JsonPathItem(item_type=JsonPathItemType.INDEX, index=index),
147-
path[right_bracket_pos + 1 :],
148-
)
149-
except ValueError as e:
150-
raise ValueError("Invalid path") from e
146+
index_str = path[left_bracket_pos + 1 : right_bracket_pos]
147+
if not (index_str.isascii() and index_str.isdigit()):
148+
raise ValueError("Invalid path")
149+
150+
index = int(index_str)
151+
if index > U64_MAX:
152+
raise ValueError("Invalid path")
153+
154+
return (
155+
JsonPathItem(item_type=JsonPathItemType.INDEX, index=index),
156+
path[right_bracket_pos + 1 :],
157+
)

‎qdrant_client/local/local_collection.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2933,11 +2933,12 @@ def delete_payload(
29332933
| models.PointIdsList
29342934
),
29352935
) -> None:
2936+
parsed_keys = [parse_json_path(key) for key in keys]
29362937
ids = self._selector_to_ids(selector)
29372938
for point_id in ids:
29382939
idx = self.ids[point_id]
2939-
for key in keys:
2940-
delete_value_by_key(self.payload[idx], parse_json_path(key))
2940+
for parsed_key in parsed_keys:
2941+
delete_value_by_key(self.payload[idx], parsed_key)
29412942
self._persist_by_id(point_id)
29422943

29432944
def clear_payload(

‎qdrant_client/local/payload_value_setter.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,7 @@ def _delete(data: Any, k_list: list[JsonPathItem]) -> None:
7272
_delete(data[current_key.key], k_list.copy())
7373
elif current_key.item_type == JsonPathItemType.INDEX:
7474
assert current_key.index is not None
75-
# The server addresses array elements with an unsigned index, so
76-
# only a non-negative, in-range index traverses; a negative or
77-
# out-of-range index is a no-op.
78-
if isinstance(data, list) and 0 <= current_key.index < len(data):
75+
if isinstance(data, list) and current_key.index < len(data):
7976
_delete(data[current_key.index], k_list.copy())
8077
elif current_key.item_type == JsonPathItemType.WILDCARD_INDEX:
8178
if isinstance(data, list):

‎qdrant_client/local/tests/test_payload_utils.py‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,25 @@ def test_parse_json_path() -> None:
183183
jp_key = "a.c[].[]"
184184
parse_json_path(jp_key)
185185

186+
# the server accepts only unsigned decimal digits that fit into u64
187+
for jp_key in (
188+
"a[-1]",
189+
"a[-1].b",
190+
"a[+1]",
191+
"a[1_0]",
192+
"a[ 1 ]",
193+
"a[\u0661]",
194+
f"a[{2**64}]",
195+
):
196+
with pytest.raises(ValueError):
197+
parse_json_path(jp_key)
198+
199+
assert parse_json_path("a[01]") == [
200+
JsonPathItem(item_type=JsonPathItemType.KEY, key="a"),
201+
JsonPathItem(item_type=JsonPathItemType.INDEX, index=1),
202+
]
203+
assert parse_json_path(f"a[{2**64 - 1}]")[1].index == 2**64 - 1
204+
186205

187206
def test_value_by_key() -> None:
188207
payload = {
@@ -460,41 +479,32 @@ def test_set_value_by_key() -> None:
460479

461480
# region exceptions
462481

463-
try:
482+
# incorrect quotes
483+
with pytest.raises(ValueError):
464484
payload = {"a": []}
465485
new_value = {"c": 3}
466486
key = "a.'b.c'"
467487
set_value_by_key(payload, parse_json_path(key), new_value)
468-
assert False, f"Should've raised an exception due to the key with incorrect quotes: {key}"
469-
except Exception:
470-
assert True
471488

472-
try:
489+
# negative indexation is not supported
490+
with pytest.raises(ValueError):
473491
payload = {"a": [{"b": 1}, {"b": 2}]}
474492
new_value = {"c": 3}
475493
key = "a[-1]"
476494
set_value_by_key(payload, parse_json_path(key), new_value)
477-
assert False, "Negative indexation is not supported"
478-
except Exception:
479-
assert True
480495

481-
try:
496+
# unbalanced brackets
497+
with pytest.raises(ValueError):
482498
payload = {"a": [{"b": 1}, {"b": 2}]}
483499
new_value = {"c": 3}
484500
key = "a["
485501
set_value_by_key(payload, parse_json_path(key), new_value)
486-
assert False, f"Should've raised an exception due to the incorrect key: {key}"
487-
except Exception:
488-
assert True
489502

490-
try:
503+
with pytest.raises(ValueError):
491504
payload = {"a": [{"b": 1}, {"b": 2}]}
492505
new_value = {"c": 3}
493506
key = "a]"
494507
set_value_by_key(payload, parse_json_path(key), new_value)
495-
assert False, f"Should've raise an exception due to the incorrect key: {key}"
496-
except Exception:
497-
assert True
498508

499509
# endregion
500510

‎tests/congruence_tests/test_payload.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,44 @@ def delete_keys(payload, keys):
119119
# element by index is not idempotent), so both sides leave it untouched
120120
delete_keys({"loc": [{"x": 1}, {"x": 2}]}, ["loc[0]"])
121121

122+
# a terminal wildcard clears the array instead
123+
delete_keys({"loc": [1, 2, 3], "top": 9}, ["loc[]"])
124+
125+
# arrays are not implicitly flattened: "loc.x" does not reach into elements,
126+
# unlike in filters, where "loc.x" and "loc[].x" are equivalent
127+
delete_keys({"loc": [{"x": 1, "y": 2}]}, ["loc.x"])
128+
129+
# a dotted path never matches a literal key containing a dot
130+
delete_keys({"a.b": 1, "a": {"b": 2}}, ["a.b"])
131+
132+
# nested wildcards clear each inner array
133+
delete_keys({"loc": [[1, 2], [3, 4]]}, ["loc[][]"])
134+
122135
# top-level and nested keys together
123136
delete_keys({"a": {"b": 1}, "top": 9}, ["a.b", "top"])
124137

138+
# an invalid json path rejects the whole request on both sides: nothing is
139+
# deleted, not even for the keys preceding the invalid one. "loc[-1]" is
140+
# invalid too: the server addresses array elements with an unsigned index.
141+
for invalid_key in ("not a valid path", "loc[-1]", "loc[-1].x"):
142+
for client in (local_client, remote_client):
143+
client.upsert(
144+
collection_name=COLLECTION_NAME,
145+
points=[
146+
PointStruct(id=9999, payload={"a": {"b": 1}, "loc": [1, 2]}, vector=vector),
147+
PointStruct(id=10000, payload={"a": {"b": 1}, "loc": [1, 2]}, vector=vector),
148+
],
149+
wait=True,
150+
)
151+
with pytest.raises((ValueError, UnexpectedResponse, grpc.RpcError)):
152+
client.delete_payload(
153+
collection_name=COLLECTION_NAME,
154+
keys=["a.b", invalid_key],
155+
points=[9999, 10000],
156+
wait=True,
157+
)
158+
compare_collections(local_client, remote_client, 2)
159+
125160

126161
@pytest.mark.parametrize("prefer_grpc", [True, False])
127162
def test_clear_payload(prefer_grpc):

‎tests/local/test_delete_value_by_key.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,8 @@ def _delete(payload: dict, key: str) -> dict:
2424
# single element by index (it is not idempotent)
2525
({"loc": [{"x": 1}, {"x": 2}]}, "loc[0]", {"loc": [{"x": 1}, {"x": 2}]}),
2626
({"loc": [1, 2, 3]}, "loc[1]", {"loc": [1, 2, 3]}),
27-
# negative indices are not representable server-side, so they must not
28-
# delete or traverse (Python-style negative indexing must be ignored)
29-
({"loc": [1, 2, 3]}, "loc[-1]", {"loc": [1, 2, 3]}),
30-
({"loc": [{"x": 1}, {"x": 2}]}, "loc[-1].x", {"loc": [{"x": 1}, {"x": 2}]}),
27+
# an out-of-range index does not traverse
28+
({"loc": [{"x": 1}]}, "loc[5].x", {"loc": [{"x": 1}]}),
3129
# array wildcard removes the field from every element
3230
(
3331
{"loc": [{"x": 1, "y": 2}, {"x": 3, "y": 4}]},
@@ -45,3 +43,11 @@ def _delete(payload: dict, key: str) -> dict:
4543
)
4644
def test_delete_value_by_key(payload, key, expected):
4745
assert _delete(payload, key) == expected
46+
47+
48+
@pytest.mark.parametrize("key", ["loc[-1]", "loc[-1].x"])
49+
def test_negative_index_is_rejected(key):
50+
# not a valid json path server-side, parse_json_path rejects it before
51+
# delete_value_by_key is reached
52+
with pytest.raises(ValueError):
53+
parse_json_path(key)

0 commit comments

Comments
 (0)