=================================== FAILURES ===================================
_____________ test_shard_wal_delta_transfer_manual_recovery_chain ______________
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-1/test_shard_wal_delta_transfer_1')
def test_shard_wal_delta_transfer_manual_recovery_chain(tmp_path: pathlib.Path):
assert_project_root()
# Prevent automatic recovery on restarted node, so we can manually recover with a specific transfer method
env={
"QDRANT__STORAGE__PERFORMANCE__INCOMING_SHARD_TRANSFERS_LIMIT": "0",
"QDRANT__STORAGE__PERFORMANCE__OUTGOING_SHARD_TRANSFERS_LIMIT": "0",
}
# seed port to reuse the same port for the restarted nodes
peer_api_uris, peer_dirs, bootstrap_uri = start_cluster(tmp_path, 5, 20000, extra_env=env)
create_collection(peer_api_uris[0], shard_number=1, replication_factor=5)
wait_collection_exists_and_active_on_all_peers(
collection_name=COLLECTION_NAME,
peer_api_uris=peer_api_uris,
)
peer_cluster_info = [get_collection_cluster_info(uri, COLLECTION_NAME) for uri in peer_api_uris]
peer_ids = [cluster_info['peer_id'] for cluster_info in peer_cluster_info]
# Start pushing points to the cluster
upload_process_1 = run_update_points_in_background(peer_api_uris[0], COLLECTION_NAME, init_offset=100000, throttle=True)
upload_process_2 = run_update_points_in_background(peer_api_uris[1], COLLECTION_NAME, init_offset=200000, throttle=True)
upload_process_3 = run_update_points_in_background(peer_api_uris[2], COLLECTION_NAME, init_offset=300000, throttle=True)
upload_process_4 = run_update_points_in_background(peer_api_uris[3], COLLECTION_NAME, init_offset=400000, throttle=True)
upload_process_5 = run_update_points_in_background(peer_api_uris[4], COLLECTION_NAME, init_offset=500000, throttle=True)
sleep(1)
# Kill 5th peer
upload_process_5.kill()
processes.pop().kill()
sleep(1)
# Kill 4th peer
upload_process_4.kill()
processes.pop().kill()
upsert_random_points(peer_api_uris[0], 100, batch_size=5)
sleep(3)
# Restart 3rd and 4th peer
peer_api_uris[3] = start_peer(peer_dirs[3], "peer_3_restarted.log", bootstrap_uri, extra_env=env)
peer_api_uris[4] = start_peer(peer_dirs[4], "peer_4_restarted.log", bootstrap_uri, extra_env=env)
wait_for_peer_online(peer_api_uris[3], "/")
wait_for_peer_online(peer_api_uris[4], "/")
# Recover shard with WAL delta transfer
r = requests.post(
f"{peer_api_uris[0]}/collections/{COLLECTION_NAME}/cluster", json={
"replicate_shard": {
"shard_id": 0,
"from_peer_id": peer_ids[0],
"to_peer_id": peer_ids[3],
"method": "wal_delta",
}
})
assert_http_ok(r)
# Assert WAL delta transfer progress, and wait for it to finish
wait_for_collection_shard_transfers_count(peer_api_uris[0], COLLECTION_NAME, 1)
wait_for_collection_shard_transfers_count(peer_api_uris[0], COLLECTION_NAME, 0)
# Start inserting into the fourth peer again
upload_process_4 = run_update_points_in_background(peer_api_uris[3], COLLECTION_NAME, init_offset=600000, throttle=True)
sleep(1)
# Recover shard with WAL delta transfer
r = requests.post(
f"{peer_api_uris[3]}/collections/{COLLECTION_NAME}/cluster", json={
"replicate_shard": {
"shard_id": 0,
"from_peer_id": peer_ids[3],
"to_peer_id": peer_ids[4],
"method": "wal_delta",
}
})
assert_http_ok(r)
# Assert WAL delta transfer progress, and wait for it to finish
wait_for_collection_shard_transfers_count(peer_api_uris[3], COLLECTION_NAME, 1)
wait_for_collection_shard_transfers_count(peer_api_uris[3], COLLECTION_NAME, 0)
upload_process_1.kill()
upload_process_2.kill()
upload_process_3.kill()
upload_process_4.kill()
sleep(1)
# All nodes must have one shard
for uri in peer_api_uris:
cluster_info = get_collection_cluster_info(uri, COLLECTION_NAME)
assert len(cluster_info['local_shards']) == 1
# Ensure data consistency
data = []
for uri in peer_api_uris:
r = requests.post(
f"{uri}/collections/{COLLECTION_NAME}/points/scroll", json={
"limit": 999999999,
"with_vectors": True,
"with_payload": True,
}
)
assert_http_ok(r)
data.append(r.json()["result"])
> check_data_consistency(data)
tests/consensus_tests/test_shard_wal_delta_transfer.py:352:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
data = [{'next_page_offset': None, 'points': [{'id': 0, 'payload': {'city': 'Moscow'}, 'vector': {'': [0.9941661, 0.06556095,...ctor': {'': [0.5849804, 0.8663159, 0.27923757, 0.2876978], 'sparse-text': {'indices': [...], 'values': [...]}}}, ...]}]
def check_data_consistency(data):
assert(len(data) > 1)
for i in range(len(data) - 1):
j = i + 1
data_i = data[i]
data_j = data[j]
if data_i != data_j:
ids_i = set(x['id'] for x in data_i["points"])
ids_j = set(x['id'] for x in data_j["points"])
diff = ids_i - ids_j
if len(diff) == 0:
i_points, j_points = [sorted(l, key=itemgetter('id')) for l in (data_i["points"], data_j["points"])]
pairs = zip(i_points, j_points)
diff = [(x, y) for x, y in pairs if x != y]
print(f'Points between {i} and {j} are not equal, diff: "{diff}"')
elif len(diff) < 100:
print(f"Diff between {i} and {j}: {diff}")
else:
print(f"Diff len between {i} and {j}: {len(diff)}")
> assert False, "Data on all nodes should be consistent"
E AssertionError: Data on all nodes should be consistent
E assert False
tests/consensus_tests/test_shard_wal_delta_transfer.py:58: AssertionError
----------------------------- Captured stdout call -----------------------------
Starting first peer with uri http://127.0.0.1:20000, http: http://localhost:20002/cluster, p2p: 20000
Could not contact peer http://127.0.0.1:20002/ to fetch cluster size
Starting follower peer with bootstrap uri http://127.0.0.1:20000, http: http://localhost:20102/cluster, p2p: 20100
Starting follower peer with bootstrap uri http://127.0.0.1:20000, http: http://localhost:20202/cluster, p2p: 20200
Starting follower peer with bootstrap uri http://127.0.0.1:20000, http: http://localhost:20302/cluster, p2p: 20300
Starting follower peer with bootstrap uri http://127.0.0.1:20000, http: http://localhost:20402/cluster, p2p: 20400
Cluster size invalid for peer http://127.0.0.1:20002/ 1/5
Starting follower peer with bootstrap uri http://127.0.0.1:20000, http: http://localhost:38817/cluster, p2p: 45257
Starting follower peer with bootstrap uri http://127.0.0.1:20000, http: http://localhost:53235/cluster, p2p: 42103
Points between 0 and 1 are not equal, diff: "[({'id': 600000, 'payload': {'city': 'Paris'}, 'vector': {'sparse-text': {'indices': [1, 4, 19, 46, 59, 67, 75, 84, 89, 100, 107, 113, 131, 137, 156, 175, 176, 205, 249, 252, 278, 301, 313, 322, 324, 338, 340, 345, 352, 354, 356, 362, 366, 373, 384, 387, 390, 398, 400, 406, 411, 414, 417, 419, 432, 435, 463, 478, 485, 487, 489, 493, 503, 511, 514, 524, 529, 530, 531, 535, 537, 555, 583, 589, 591, 599, 604, 609, 617, 621, 622, 645, 647, 666, 676, 693, 706, 711, 712, 734, 767, 770, 771, 776, 782, 795, 796, 799, 817, 824, 843, 861, 865, 871, 872, 891, 910, 921, 952, 967], 'values': [0.789072, 0.463268, 0.769485, 0.717054, 0.853414, 0.712039, 0.441913, 0.600267, 0.699079, 0.438773, 0.479047, 0.500992, 0.852767, 0.0509, 0.74747, 0.393006, 0.241968, 0.375223, 0.135356, 0.013795, 0.688446, 0.695545, 0.381236, 0.988824, 0.057601, 0.348714, 0.806404, 0.606361, 0.014561, 0.306279, 0.553721, 0.394709, 0.255397, 0.63444, 0.503153, 0.571468, 0.240177, 0.891053, 0.830577, 0.492042, 0.483978, 0.304068, 0.965333, 0.967742, 0.497149, 0.820279, 0.215052, 0.493846, 0.228001, 0.386657, 0.609302, 0.031087, 0.099987, 0.816315, 0.763487, 0.705903, 0.015021, 0.655978, 0.351192, 0.869357, 0.454211, 0.498113, 0.124999, 0.671943, 0.206512, 0.525413, 0.927818, 0.876234, 0.323176, 0.756671, 0.688768, 0.763183, 0.752937, 0.544458, 0.396697, 0.296646, 0.42844, 0.124989, 0.591817, 0.18219, 0.889562, 0.145266, 0.712238, 0.748456, 0.505401, 0.715388, 0.128557, 0.32194, 0.038118, 0.670386, 0.006255, 0.289771, 0.58656, 0.590224, 0.005136, 0.347397, 0.559869, 0.287629, 0.191461, 0.384873]}, '': [0.39540964, 0.6323538, 0.36564022, 0.30840683]}}, {'id': 500006, 'payload': {'city': 'Moscow'}, 'vector': {'sparse-text': {'indices': [15, 29, 40, 46, 50, 52, 63, 65, 81, 99, 104, 120, 122, 159, 164, 165, 170, 180, 185, 198, 211, 217, 220, 224, 232, 240, 244, 256, 257, 265, 275, 295, 301, 306, 316, 323, 353, 378, 389, 396, 402, 404, 407, 423, 456, 469, 496, 500, 534, 540, 543, 547, 551, 560, 568, 569, 583, 588, 591, 593, 600, 616, 619, 633, 642, 670, 703, 704, 708, 721, 730, 732, 739, 751, 765, 766, 771, 778, 811, 829, 849, 882, 905, 906, 908, 910, 911, 922, 923, 926, 932, 953, 961, 962, 963, 965, 978, 980, 991, 998], 'values': [0.412594, 0.17176, 0.786812, 0.054851, 0.140884, 0.281079, 0.190932, 0.177784, 0.12941, 0.882401, 0.176027, 0.737437, 0.684079, 0.136422, 0.188236, 0.05736, 0.989005, 0.190417, 0.166875, 0.762061, 0.505687, 0.485822, 0.602178, 0.054455, 0.064788, 0.585227, 0.795551, 0.147019, 0.734483, 0.260897, 0.402426, 0.139923, 0.49009, 0.450084, 0.14884, 0.517932, 0.110609, 0.482537, 0.098598, 0.642648, 0.208783, 0.634552, 0.683261, 0.826857, 0.491984, 0.267916, 0.406788, 0.623033, 0.133474, 0.692093, 0.039173, 0.036964, 0.118201, 0.270156, 0.827337, 0.676743, 0.332216, 0.839104, 0.322413, 0.491819, 0.329362, 0.682205, 0.018184, 0.01931, 0.002391, 0.888405, 0.244481, 0.875378, 0.865482, 0.367281, 0.884237, 0.881776, 0.188914, 0.595973, 0.063485, 0.362942, 0.670149, 0.008668, 0.645131, 0.756366, 0.555004, 0.584889, 0.312072, 0.414861, 0.080818, 0.756513, 0.116967, 0.273365, 0.433659, 0.641166, 0.889953, 0.947592, 0.694363, 0.820581, 0.09856, 0.706954, 0.335077, 0.260191, 0.062493, 0.168953]}, '': [0.8831262, 0.10412683, 0.42271966, 0.16833197]}}), ({'id': 600001, 'payload': {'city': 'Rome'}, 'vector': {'': [0.79504794, 0.2618923, 0.38180995, 0.9233551], 'sparse-text': {'indices': [3, 13, 38, 39, 48, 50, 51, 60, 67, 74, 82, 88, 89, 101, 113, 118, 143, 149, 151, 152, 157, 159, 173, 183, 208, 217, 219, 224, 234, 245, 247, 254, 260, 272, 276, 286, 306, 308, 335, 342, 356, 372, 378, 384, 397, 400, 413, 414, 430, 449, 452, 454, 462, 463, 465, 469, 473, 475, 496, 502, 513, 520, 523, 537, 547, 561, 576, 580, 591, 596, 620, 622, 626, 633, 667, 688, 703, 727, 735, 746, 763, 770, 781, 792, 808, 812, 828, 832, 843, 872, 918, 932, 933, 936, 940, 944, 946, 951, 981, 986], 'values': [0.943567, 0.77723, 0.136811, 0.252933, 0.741542, 0.264805, 0.302775, 0.446093, 0.539731, 0.015395, 0.176784, 0.654775, 0.128241, 0.275, 0.132494, 0.782695, 0.655432, 0.130584, 0.051878, 0.555516, 0.459457, 0.973147, 0.46811, 0.605735, 0.838719, 0.795299, 0.3381, 0.600993, 0.758588, 0.440242, 0.06223, 0.450277, 0.501287, 0.049653, 0.558512, 0.405785, 0.439449, 0.182311, 0.458171, 0.654708, 0.834741, 0.99524, 0.279969, 0.665792, 0.46513, 0.963697, 0.349047, 0.199781, 0.000656, 0.697745, 0.191683, 0.163199, 0.076703, 0.002687, 0.78199, 0.810182, 0.855132, 0.809195, 0.150024, 0.543101, 0.617198, 0.628506, 0.619095, 0.265767, 0.032954, 0.265288, 0.934105, 0.614146, 0.700221, 0.839354, 0.037722, 0.426507, 0.284026, 0.843998, 0.453675, 0.746363, 0.129857, 0.636516, 0.970787, 0.192541, 0.257243, 0.225995, 0.619929, 0.416588, 0.627821, 0.483043, 0.721086, 0.277383, 0.595715, 0.599835, 0.121173, 0.788263, 0.004385, 0.157389, 0.793507, 0.241103, 0.893627, 0.807122, 0.007778, 0.435229]}}}, {'id': 500007, 'payload': {'city': 'Rome'}, 'vector': {'': [0.9612023, 0.7965315, 0.6139302, 0.1547004], 'sparse-text': {'indices': [13, 20, 22, 30, 33, 38, 44, 46, 47, 75, 76, 93, 96, 98, 111, 126, 130, 137, 139, 143, 160, 164, 167, 191, 214, 222, 230, 232, 234, 241, 254, 257, 267, 276, 284, 285, 288, 295, 322, 331, 345, 347, 351, 353, 360, 418, 436, 442, 449, 451, 464, 471, 488, 501, 503, 505, 516, 530, 532, 546, 561, 566, 571, 589, 607, 627, 637, 641, 642, 652, 661, 674, 715, 720, 721, 723, 749, 757, 765, 767, 770, 773, 774, 778, 807, 808, 811, 816, 853, 854, 863, 864, 898, 943, 967, 973, 977, 987, 988, 989], 'values': [0.868156, 0.441588, 0.211637, 0.151214, 0.917313, 0.067735, 0.816363, 0.038335, 0.848648, 0.96667, 0.888054, 0.435043, 0.03222, 0.277149, 0.297057, 0.981978, 0.815682, 0.262162, 0.854034, 0.724485, 0.653052, 0.495174, 0.762701, 0.184079, 0.330659, 0.85109, 0.95087, 0.228509, 0.216625, 0.580997, 0.757618, 0.297763, 0.796522, 0.125569, 0.619901, 0.470866, 0.222061, 0.433484, 0.194595, 0.70581, 0.330225, 0.160654, 0.833478, 0.394658, 0.217308, 0.35551, 0.701487, 0.842896, 0.256417, 0.283496, 0.845853, 0.633673, 0.219623, 0.622436, 0.747478, 0.234775, 0.5892, 0.815517, 0.906053, 0.45844, 0.230969, 0.69833, 0.16072, 0.686269, 0.713999, 0.60858, 0.12911, 0.824354, 0.540815, 0.051433, 0.097841, 0.643666, 0.078045, 0.943245, 0.395296, 0.144702, 0.606859, 0.27021, 0.205303, 0.545418, 0.514257, 0.772279, 0.800838, 0.971492, 0.258121, 0.945603, 0.524675, 0.143043, 0.721411, 0.89831, 0.820553, 0.350103, 0.361837, 0.475663, 0.096021, 0.243688, 0.213685, 0.318879, 0.343755, 0.384808]}}}), ({'id': 600002, 'payload': {'city': 'Paris'}, 'vector': {'sparse-text': {'indices': [2, 22, 24, 35, 45, 55, 58, 71, 97, 103, 112, 126, 127, 151, 153, 175, 207, 214, 222, 231, 232, 236, 241, 253, 272, 279, 280, 282, 312, 335, 339, 341, 342, 350, 358, 362, 371, 391, 402, 410, 411, 412, 417, 430, 444, 450, 456, 485, 511, 514, 516, 525, 526, 533, 534, 543, 548, 579, 586, 591, 597, 599, 606, 626, 630, 635, 640, 644, 654, 662, 685, 690, 705, 715, 724, 733, 734, 752, 765, 773, 781, 788, 804, 808, 811, 848, 854, 870, 878, 891, 895, 901, 914, 916, 931, 934, 946, 948, 949, 987], 'values': [0.00051, 0.562466, 0.940418, 0.976624, 0.2594, 0.615006, 0.123271, 0.782648, 0.992505, 0.159123, 0.990525, 0.549185, 0.872058, 0.171861, 0.442934, 0.442748, 0.161366, 0.614578, 0.148331, 0.244895, 0.043024, 0.700474, 0.82261, 0.698646, 0.169462, 0.786864, 0.997871, 0.667552, 0.056694, 0.049394, 0.487395, 0.402988, 0.86772, 0.599197, 0.642781, 0.630191, 0.615141, 0.748896, 0.19456, 0.065897, 0.58821, 0.783153, 0.550291, 0.061237, 0.3957, 0.649114, 0.030537, 0.834276, 0.871608, 0.6019, 0.995402, 0.580817, 0.579654, 0.719119, 0.498984, 0.495195, 0.685694, 0.30868, 0.268336, 0.13683, 0.170691, 0.918677, 0.753212, 0.626444, 0.214505, 0.89806, 0.122058, 0.92985, 0.847015, 0.942409, 0.187842, 0.26031, 0.831778, 0.795854, 0.288971, 0.822011, 0.211268, 0.192235, 0.471094, 0.007178, 0.995451, 0.041261, 0.119365, 0.607167, 0.137105, 0.963954, 0.3151, 0.595685, 0.453736, 0.559733, 0.592392, 0.745336, 0.203798, 0.844072, 0.541066, 0.162388, 0.347115, 0.742375, 0.156928, 0.292072]}, '': [0.1779737, 0.14758846, 0.014648082, 0.31925845]}}, {'id': 500008, 'payload': {'city': 'Rome'}, 'vector': {'': [0.50367427, 0.5417227, 0.08472868, 0.76834357], 'sparse-text': {'indices': [17, 25, 39, 45, 59, 71, 79, 80, 83, 91, 105, 124, 137, 149, 151, 163, 166, 169, 175, 185, 191, 202, 212, 215, 222, 231, 239, 244, 270, 273, 289, 291, 296, 299, 306, 329, 335, 338, 345, 346, 347, 354, 362, 370, 392, 401, 408, 446, 467, 485, 503, 507, 510, 525, 534, 553, 560, 582, 595, 602, 624, 635, 636, 649, 661, 663, 676, 680, 696, 697, 715, 717, 718, 724, 726, 727, 745, 755, 767, 777, 795, 815, 835, 847, 856, 859, 876, 877, 879, 890, 896, 912, 929, 938, 955, 957, 971, 975, 996, 998], 'values': [0.435366, 0.272593, 0.653438, 0.147865, 0.893833, 0.958522, 0.528964, 0.611684, 0.669604, 0.116846, 0.822383, 0.06515, 0.674193, 0.644196, 0.980802, 0.472036, 0.98031, 0.39712, 0.16581, 0.734541, 0.506331, 0.035119, 0.676613, 0.970655, 0.939587, 0.921908, 0.215019, 0.958204, 0.287247, 0.294085, 0.007947, 0.175546, 0.838074, 0.24913, 0.649679, 0.981445, 0.922488, 0.122357, 0.058332, 0.694556, 0.407277, 0.148938, 0.759413, 0.875166, 0.075787, 0.92106, 0.992323, 0.986317, 0.903546, 0.995326, 0.704688, 0.602284, 0.477451, 0.773361, 0.713705, 0.668767, 0.148093, 0.268164, 0.527812, 0.203962, 0.049073, 0.156009, 0.457984, 0.239255, 0.883282, 0.237383, 0.266558, 0.790756, 0.308162, 0.694619, 0.45251, 0.911574, 0.520713, 0.272583, 0.249755, 0.335656, 0.270251, 0.259441, 0.517081, 0.453529, 0.777269, 0.042425, 0.834485, 0.614703, 0.768375, 0.82283, 0.318197, 0.757629, 0.269898, 0.451404, 0.375211, 0.532462, 0.801153, 0.074207, 0.869573, 0.253559, 0.514512, 0.256591, 0.17407, 0.854538]}}}), ({'id': 600003, 'payload': {'city': 'London'}, 'vector': {'': [0.9988849, 0.26974395, 0.6029054, 0.6568519], 'sparse-text': {'indices': [0, 5, 9, 10, 16, 21, 24, 32, 33, 35, 36, 40, 58, 68, 80, 103, 104, 109, 115, 117, 121, 128, 136, 139, 143, 146, 155, 156, 164, 195, 198, 222, 238, 291, 310, 311, 326, 330, 344, 346, 356, 370, 396, 399, 406, 408, 412, 418, 435, 472, 483, 490, 497, 501, 507, 519, 537, 545, 546, 563, 579, 596, 603, 628, 641, 643, 653, 657, 690, 691, 699, 702, 709, 713, 723, 739, 751, 761, 774, 791, 817, 818, 820, 840, 842, 846, 853, 855, 858, 874, 876, 901, 913, 919, 934, 939, 953, 980, 983, 996], 'values': [0.310837, 0.752326, 0.524747, 0.917056, 0.754639, 0.252396, 0.725527, 0.428409, 0.083151, 0.880134, 0.731397, 0.309989, 0.584028, 0.963609, 0.89092, 0.317998, 0.397367, 0.983126, 0.901562, 0.188894, 0.447685, 0.296088, 0.439627, 0.870113, 0.459559, 0.751588, 0.641047, 0.152395, 0.997534, 0.549155, 0.055907, 0.290001, 0.372604, 0.682439, 0.905666, 0.968786, 0.609804, 0.535874, 0.590461, 0.657529, 0.196087, 0.283734, 0.578764, 0.891285, 0.453527, 0.522026, 0.485266, 0.50876, 0.902933, 0.929648, 0.096882, 0.872508, 0.035917, 0.697746, 0.758176, 0.060231, 0.21804, 0.674419, 0.974596, 0.934961, 0.398503, 0.845054, 0.942997, 0.981555, 0.027524, 0.029811, 0.788399, 0.646528, 0.461913, 0.924405, 0.235847, 0.471128, 0.111758, 0.292536, 0.33374, 0.888349, 0.880249, 0.242507, 0.725144, 0.618165, 0.323125, 0.652414, 0.728593, 0.501224, 0.900786, 0.521344, 0.184088, 0.135407, 0.317977, 0.4733, 0.15384, 0.104516, 0.459453, 0.965347, 0.349584, 0.441891, 0.990629, 0.47087, 0.807296, 0.355724]}}}, {'id': 600000, 'payload': {'city': 'Paris'}, 'vector': {'sparse-text': {'indices': [1, 4, 19, 46, 59, 67, 75, 84, 89, 100, 107, 113, 131, 137, 156, 175, 176, 205, 249, 252, 278, 301, 313, 322, 324, 338, 340, 345, 352, 354, 356, 362, 366, 373, 384, 387, 390, 398, 400, 406, 411, 414, 417, 419, 432, 435, 463, 478, 485, 487, 489, 493, 503, 511, 514, 524, 529, 530, 531, 535, 537, 555, 583, 589, 591, 599, 604, 609, 617, 621, 622, 645, 647, 666, 676, 693, 706, 711, 712, 734, 767, 770, 771, 776, 782, 795, 796, 799, 817, 824, 843, 861, 865, 871, 872, 891, 910, 921, 952, 967], 'values': [0.789072, 0.463268, 0.769485, 0.717054, 0.853414, 0.712039, 0.441913, 0.600267, 0.699079, 0.438773, 0.479047, 0.500992, 0.852767, 0.0509, 0.74747, 0.393006, 0.241968, 0.375223, 0.135356, 0.013795, 0.688446, 0.695545, 0.381236, 0.988824, 0.057601, 0.348714, 0.806404, 0.606361, 0.014561, 0.306279, 0.553721, 0.394709, 0.255397, 0.63444, 0.503153, 0.571468, 0.240177, 0.891053, 0.830577, 0.492042, 0.483978, 0.304068, 0.965333, 0.967742, 0.497149, 0.820279, 0.215052, 0.493846, 0.228001, 0.386657, 0.609302, 0.031087, 0.099987, 0.816315, 0.763487, 0.705903, 0.015021, 0.655978, 0.351192, 0.869357, 0.454211, 0.498113, 0.124999, 0.671943, 0.206512, 0.525413, 0.927818, 0.876234, 0.323176, 0.756671, 0.688768, 0.763183, 0.752937, 0.544458, 0.396697, 0.296646, 0.42844, 0.124989, 0.591817, 0.18219, 0.889562, 0.145266, 0.712238, 0.748456, 0.505401, 0.715388, 0.128557, 0.32194, 0.038118, 0.670386, 0.006255, 0.289771, 0.58656, 0.590224, 0.005136, 0.347397, 0.559869, 0.287629, 0.191461, 0.384873]}, '': [0.39540964, 0.6323538, 0.36564022, 0.30840683]}}), ({'id': 600004, 'payload': {'city': 'Rome'}, 'vector': {'': [0.4998061, 0.3031299, 0.24907926, 0.51130885], 'sparse-text': {'indices': [1, 31, 59, 64, 77, 99, 105, 115, 125, 126, 135, 137, 154, 156, 162, 163, 183, 188, 191, 204, 205, 208, 215, 225, 238, 242, 248, 262, 274, 281, 284, 290, 296, 298, 303, 325, 329, 334, 339, 351, 364, 374, 392, 393, 399, 414, 419, 426, 443, 449, 453, 478, 480, 494, 501, 508, 525, 540, 551, 553, 563, 564, 567, 596, 598, 603, 615, 628, 635, 646, 658, 695, 713, 719, 737, 748, 773, 779, 781, 799, 803, 812, 813, 817, 839, 854, 857, 860, 891, 927, 941, 948, 950, 953, 961, 978, 986, 989, 994, 996], 'values': [0.615694, 0.300912, 0.386504, 0.258277, 0.779573, 0.343234, 0.746103, 0.745005, 0.426405, 0.763797, 0.852487, 0.230006, 0.271749, 0.976216, 0.243602, 0.873859, 0.604074, 0.244074, 0.188886, 0.627372, 0.570836, 0.005186, 0.784776, 0.538444, 0.656657, 0.269087, 0.990319, 0.086499, 0.771444, 0.351756, 0.397723, 0.566866, 0.537824, 0.87322, 0.719801, 0.86932, 0.561208, 0.999826, 0.384467, 0.047865, 0.460656, 0.597939, 0.482658, 0.646689, 0.279666, 0.667332, 0.054421, 0.18873, 0.481638, 0.229469, 0.799368, 0.35266, 0.097812, 0.896007, 0.991015, 0.806829, 0.657068, 0.984498, 0.359541, 0.806695, 0.221665, 0.874126, 0.731902, 0.718308, 0.622315, 0.240101, 0.355551, 0.832708, 0.366298, 0.887952, 0.94767, 0.204265, 0.059249, 0.752233, 0.012864, 0.345209, 0.630206, 0.346724, 0.785567, 0.098856, 0.667917, 0.208617, 0.493264, 0.180217, 0.46185, 0.91762, 0.304266, 0.128658, 0.827231, 0.757635, 0.291425, 0.994569, 0.246294, 0.286599, 0.558524, 0.916878, 0.844759, 0.354552, 0.400621, 0.845335]}}}, {'id': 600001, 'payload': {'city': 'Rome'}, 'vector': {'sparse-text': {'indices': [3, 13, 38, 39, 48, 50, 51, 60, 67, 74, 82, 88, 89, 101, 113, 118, 143, 149, 151, 152, 157, 159, 173, 183, 208, 217, 219, 224, 234, 245, 247, 254, 260, 272, 276, 286, 306, 308, 335, 342, 356, 372, 378, 384, 397, 400, 413, 414, 430, 449, 452, 454, 462, 463, 465, 469, 473, 475, 496, 502, 513, 520, 523, 537, 547, 561, 576, 580, 591, 596, 620, 622, 626, 633, 667, 688, 703, 727, 735, 746, 763, 770, 781, 792, 808, 812, 828, 832, 843, 872, 918, 932, 933, 936, 940, 944, 946, 951, 981, 986], 'values': [0.943567, 0.77723, 0.136811, 0.252933, 0.741542, 0.264805, 0.302775, 0.446093, 0.539731, 0.015395, 0.176784, 0.654775, 0.128241, 0.275, 0.132494, 0.782695, 0.655432, 0.130584, 0.051878, 0.555516, 0.459457, 0.973147, 0.46811, 0.605735, 0.838719, 0.795299, 0.3381, 0.600993, 0.758588, 0.440242, 0.06223, 0.450277, 0.501287, 0.049653, 0.558512, 0.405785, 0.439449, 0.182311, 0.458171, 0.654708, 0.834741, 0.99524, 0.279969, 0.665792, 0.46513, 0.963697, 0.349047, 0.199781, 0.000656, 0.697745, 0.191683, 0.163199, 0.076703, 0.002687, 0.78199, 0.810182, 0.855132, 0.809195, 0.150024, 0.543101, 0.617198, 0.628506, 0.619095, 0.265767, 0.032954, 0.265288, 0.934105, 0.614146, 0.700221, 0.839354, 0.037722, 0.426507, 0.284026, 0.843998, 0.453675, 0.746363, 0.129857, 0.636516, 0.970787, 0.192541, 0.257243, 0.225995, 0.619929, 0.416588, 0.627821, 0.483043, 0.721086, 0.277383, 0.595715, 0.599835, 0.121173, 0.788263, 0.004385, 0.157389, 0.793507, 0.241103, 0.893627, 0.807122, 0.007778, 0.435229]}, '': [0.79504794, 0.2618923, 0.38180995, 0.9233551]}}), ({'id': 600005, 'payload': {'city': 'Berlin'}, 'vector': {'': [0.81253946, 0.20593515, 0.124255456, 0.6036452], 'sparse-text': {'indices': [6, 25, 27, 40, 45, 52, 59, 74, 76, 85, 90, 98, 104, 109, 121, 152, 154, 156, 161, 163, 167, 180, 208, 219, 228, 257, 278, 281, 282, 285, 294, 296, 298, 307, 308, 310, 314, 318, 333, 352, 358, 387, 403, 426, 439, 454, 461, 465, 480, 496, 499, 501, 508, 525, 535, 537, 545, 548, 552, 566, 571, 572, 576, 579, 585, 590, 608, 613, 615, 617, 635, 644, 655, 658, 667, 682, 685, 713, 718, 746, 751, 768, 775, 802, 824, 825, 831, 834, 847, 850, 864, 867, 878, 894, 896, 920, 941, 949, 971, 984], 'values': [0.006099, 0.041014, 0.595597, 0.924165, 0.997881, 0.433743, 0.028598, 0.63686, 0.592393, 0.783584, 0.62471, 0.424795, 0.469653, 0.048767, 0.838689, 0.857366, 0.832501, 0.462223, 0.363119, 0.848415, 0.758556, 0.203476, 0.583074, 0.102271, 0.431838, 0.086634, 0.903478, 0.777956, 0.06073, 0.767535, 0.886442, 0.981458, 0.646013, 0.641629, 0.042603, 0.530816, 0.389091, 0.16402, 0.207283, 0.476958, 0.601659, 0.080345, 0.76917, 0.810417, 0.604566, 0.92758, 0.863258, 0.849647, 0.144949, 0.359851, 0.726493, 0.025558, 0.36306, 0.480009, 0.4135, 0.401027, 0.258937, 0.715953, 0.378162, 0.86779, 0.042396, 0.632647, 0.769226, 0.543659, 0.774324, 0.905226, 0.16748, 0.198984, 0.00918, 0.842955, 0.700729, 0.284272, 0.296666, 0.551427, 0.823913, 0.072204, 0.558331, 0.510899, 0.364274, 0.853089, 0.548604, 0.183164, 0.226698, 0.324472, 0.329742, 0.534086, 0.035003, 0.368365, 0.271417, 0.417472, 0.115758, 0.914352, 0.128239, 0.311827, 0.770231, 0.899918, 0.299926, 0.064528, 0.545202, 0.298211]}}}, {'id': 600002, 'payload': {'city': 'Paris'}, 'vector': {'': [0.1779737, 0.14758846, 0.014648082, 0.31925845], 'sparse-text': {'indices': [2, 22, 24, 35, 45, 55, 58, 71, 97, 103, 112, 126, 127, 151, 153, 175, 207, 214, 222, 231, 232, 236, 241, 253, 272, 279, 280, 282, 312, 335, 339, 341, 342, 350, 358, 362, 371, 391, 402, 410, 411, 412, 417, 430, 444, 450, 456, 485, 511, 514, 516, 525, 526, 533, 534, 543, 548, 579, 586, 591, 597, 599, 606, 626, 630, 635, 640, 644, 654, 662, 685, 690, 705, 715, 724, 733, 734, 752, 765, 773, 781, 788, 804, 808, 811, 848, 854, 870, 878, 891, 895, 901, 914, 916, 931, 934, 946, 948, 949, 987], 'values': [0.00051, 0.562466, 0.940418, 0.976624, 0.2594, 0.615006, 0.123271, 0.782648, 0.992505, 0.159123, 0.990525, 0.549185, 0.872058, 0.171861, 0.442934, 0.442748, 0.161366, 0.614578, 0.148331, 0.244895, 0.043024, 0.700474, 0.82261, 0.698646, 0.169462, 0.786864, 0.997871, 0.667552, 0.056694, 0.049394, 0.487395, 0.402988, 0.86772, 0.599197, 0.642781, 0.630191, 0.615141, 0.748896, 0.19456, 0.065897, 0.58821, 0.783153, 0.550291, 0.061237, 0.3957, 0.649114, 0.030537, 0.834276, 0.871608, 0.6019, 0.995402, 0.580817, 0.579654, 0.719119, 0.498984, 0.495195, 0.685694, 0.30868, 0.268336, 0.13683, 0.170691, 0.918677, 0.753212, 0.626444, 0.214505, 0.89806, 0.122058, 0.92985, 0.847015, 0.942409, 0.187842, 0.26031, 0.831778, 0.795854, 0.288971, 0.822011, 0.211268, 0.192235, 0.471094, 0.007178, 0.995451, 0.041261, 0.119365, 0.607167, 0.137105, 0.963954, 0.3151, 0.595685, 0.453736, 0.559733, 0.592392, 0.745336, 0.203798, 0.844072, 0.541066, 0.162388, 0.347115, 0.742375, 0.156928, 0.292072]}}}), ({'id': 600006, 'payload': {'city': 'Berlin'}, 'vector': {'sparse-text': {'indices': [9, 35, 57, 60, 64, 74, 106, 139, 145, 157, 164, 176, 178, 214, 218, 242, 243, 249, 250, 264, 277, 292, 322, 329, 342, 376, 387, 401, 402, 411, 418, 422, 429, 459, 472, 476, 482, 486, 490, 491, 503, 504, 530, 534, 543, 546, 562, 565, 568, 576, 581, 587, 593, 602, 607, 609, 611, 613, 617, 631, 635, 654, 660, 666, 667, 674, 697, 706, 707, 717, 731, 733, 751, 767, 800, 803, 805, 808, 812, 815, 818, 820, 825, 836, 858, 862, 871, 879, 891, 906, 908, 912, 919, 921, 925, 939, 940, 942, 952, 998], 'values': [0.665641, 0.348418, 0.920225, 0.304783, 0.644, 0.244778, 0.768284, 0.308307, 0.368344, 0.95178, 0.744845, 0.831767, 0.419561, 0.944551, 0.831735, 0.360605, 0.343198, 0.064876, 0.605045, 0.818796, 0.018277, 0.333566, 0.572912, 0.262463, 0.630656, 0.484249, 0.260256, 0.57141, 0.762597, 0.272381, 0.02535, 0.267866, 0.14959, 0.10948, 0.676837, 0.999768, 0.573049, 0.82889, 0.498438, 0.155838, 0.240857, 0.217347, 0.673608, 0.545291, 0.781386, 0.252265, 0.043313, 0.570599, 0.628222, 0.518632, 0.751526, 0.233457, 0.136045, 0.995537, 0.802237, 0.326649, 0.988772, 0.471338, 0.627995, 0.420165, 0.757587, 0.366405, 0.215607, 0.519351, 0.721701, 0.046504, 0.73863, 0.298446, 0.333906, 0.369888, 0.683933, 0.727406, 0.233519, 0.648021, 0.081425, 0.651873, 0.421883, 0.464844, 0.755799, 0.052183, 0.777377, 0.038007, 0.221064, 0.964171, 0.599638, 0.839712, 0.255356, 0.703824, 0.68043, 0.699903, 0.406731, 0.142973, 0.311669, 0.483278, 0.163661, 0.146876, 0.212925, 0.330301, 0.774281, 0.770082]}, '': [0.05512371, 0.62330705, 0.43790078, 0.89942074]}}, {'id': 600003, 'payload': {'city': 'London'}, 'vector': {'': [0.9988849, 0.26974395, 0.6029054, 0.6568519], 'sparse-text': {'indices': [0, 5, 9, 10, 16, 21, 24, 32, 33, 35, 36, 40, 58, 68, 80, 103, 104, 109, 115, 117, 121, 128, 136, 139, 143, 146, 155, 156, 164, 195, 198, 222, 238, 291, 310, 311, 326, 330, 344, 346, 356, 370, 396, 399, 406, 408, 412, 418, 435, 472, 483, 490, 497, 501, 507, 519, 537, 545, 546, 563, 579, 596, 603, 628, 641, 643, 653, 657, 690, 691, 699, 702, 709, 713, 723, 739, 751, 761, 774, 791, 817, 818, 820, 840, 842, 846, 853, 855, 858, 874, 876, 901, 913, 919, 934, 939, 953, 980, 983, 996], 'values': [0.310837, 0.752326, 0.524747, 0.917056, 0.754639, 0.252396, 0.725527, 0.428409, 0.083151, 0.880134, 0.731397, 0.309989, 0.584028, 0.963609, 0.89092, 0.317998, 0.397367, 0.983126, 0.901562, 0.188894, 0.447685, 0.296088, 0.439627, 0.870113, 0.459559, 0.751588, 0.641047, 0.152395, 0.997534, 0.549155, 0.055907, 0.290001, 0.372604, 0.682439, 0.905666, 0.968786, 0.609804, 0.535874, 0.590461, 0.657529, 0.196087, 0.283734, 0.578764, 0.891285, 0.453527, 0.522026, 0.485266, 0.50876, 0.902933, 0.929648, 0.096882, 0.872508, 0.035917, 0.697746, 0.758176, 0.060231, 0.21804, 0.674419, 0.974596, 0.934961, 0.398503, 0.845054, 0.942997, 0.981555, 0.027524, 0.029811, 0.788399, 0.646528, 0.461913, 0.924405, 0.235847, 0.471128, 0.111758, 0.292536, 0.33374, 0.888349, 0.880249, 0.242507, 0.725144, 0.618165, 0.323125, 0.652414, 0.728593, 0.501224, 0.900786, 0.521344, 0.184088, 0.135407, 0.317977, 0.4733, 0.15384, 0.104516, 0.459453, 0.965347, 0.349584, 0.441891, 0.990629, 0.47087, 0.807296, 0.355724]}}}), ({'id': 600007, 'payload': {'city': 'London'}, 'vector': {'': [0.033625696, 0.94703394, 0.077850044, 0.58111423], 'sparse-text': {'indices': [9, 12, 21, 31, 35, 40, 62, 75, 84, 115, 119, 130, 139, 145, 148, 168, 193, 227, 230, 231, 238, 243, 253, 266, 269, 270, 286, 295, 297, 303, 314, 318, 321, 339, 349, 350, 355, 360, 367, 368, 378, 395, 398, 401, 411, 470, 473, 475, 478, 524, 526, 545, 546, 551, 559, 561, 563, 575, 580, 587, 590, 599, 601, 619, 642, 649, 650, 651, 653, 672, 675, 683, 697, 704, 715, 732, 736, 743, 745, 746, 771, 774, 778, 784, 785, 786, 792, 809, 821, 822, 859, 915, 927, 953, 969, 970, 974, 978, 984, 994], 'values': [0.917857, 0.982746, 0.78812, 0.903924, 0.358587, 0.760152, 0.133249, 0.14833, 0.220425, 0.075314, 0.293215, 0.965109, 0.42874, 0.087525, 0.317281, 0.327753, 0.804944, 0.029775, 0.118523, 0.714002, 0.390026, 0.37008, 0.016912, 0.823931, 0.255645, 0.536204, 0.136131, 0.558526, 0.615286, 0.793494, 0.449356, 0.533933, 0.415334, 0.11821, 0.052219, 0.180582, 0.914738, 0.96509, 0.012821, 0.681724, 0.872383, 0.281288, 0.807585, 0.183325, 0.467274, 0.13346, 0.371189, 0.439168, 0.18781, 0.545961, 0.128614, 0.916686, 0.805591, 0.6815, 0.267806, 0.03314, 0.500474, 0.300532, 0.707826, 0.229606, 0.279991, 0.036378, 0.900211, 0.676212, 0.257466, 0.609909, 0.938757, 0.976938, 0.344318, 0.842274, 0.421745, 0.969104, 0.062404, 0.62196, 0.798657, 0.990517, 0.665366, 0.562486, 0.84874, 0.137503, 0.803548, 0.195051, 0.658588, 0.025601, 0.233617, 0.86609, 0.67359, 0.025783, 0.716486, 0.700585, 0.564038, 0.500688, 0.390762, 0.98823, 0.925078, 0.045717, 0.538811, 0.72606, 0.235757, 0.573134]}}}, {'id': 600004, 'payload': {'city': 'Rome'}, 'vector': {'': [0.4998061, 0.3031299, 0.24907926, 0.51130885], 'sparse-text': {'indices': [1, 31, 59, 64, 77, 99, 105, 115, 125, 126, 135, 137, 154, 156, 162, 163, 183, 188, 191, 204, 205, 208, 215, 225, 238, 242, 248, 262, 274, 281, 284, 290, 296, 298, 303, 325, 329, 334, 339, 351, 364, 374, 392, 393, 399, 414, 419, 426, 443, 449, 453, 478, 480, 494, 501, 508, 525, 540, 551, 553, 563, 564, 567, 596, 598, 603, 615, 628, 635, 646, 658, 695, 713, 719, 737, 748, 773, 779, 781, 799, 803, 812, 813, 817, 839, 854, 857, 860, 891, 927, 941, 948, 950, 953, 961, 978, 986, 989, 994, 996], 'values': [0.615694, 0.300912, 0.386504, 0.258277, 0.779573, 0.343234, 0.746103, 0.745005, 0.426405, 0.763797, 0.852487, 0.230006, 0.271749, 0.976216, 0.243602, 0.873859, 0.604074, 0.244074, 0.188886, 0.627372, 0.570836, 0.005186, 0.784776, 0.538444, 0.656657, 0.269087, 0.990319, 0.086499, 0.771444, 0.351756, 0.397723, 0.566866, 0.537824, 0.87322, 0.719801, 0.86932, 0.561208, 0.999826, 0.384467, 0.047865, 0.460656, 0.597939, 0.482658, 0.646689, 0.279666, 0.667332, 0.054421, 0.18873, 0.481638, 0.229469, 0.799368, 0.35266, 0.097812, 0.896007, 0.991015, 0.806829, 0.657068, 0.984498, 0.359541, 0.806695, 0.221665, 0.874126, 0.731902, 0.718308, 0.622315, 0.240101, 0.355551, 0.832708, 0.366298, 0.887952, 0.94767, 0.204265, 0.059249, 0.752233, 0.012864, 0.345209, 0.630206, 0.346724, 0.785567, 0.098856, 0.667917, 0.208617, 0.493264, 0.180217, 0.46185, 0.91762, 0.304266, 0.128658, 0.827231, 0.757635, 0.291425, 0.994569, 0.246294, 0.286599, 0.558524, 0.916878, 0.844759, 0.354552, 0.400621, 0.845335]}}}), ({'id': 600008, 'payload': {'city': 'Tokyo'}, 'vector': {'sparse-text': {'indices': [9, 10, 28, 33, 39, 54, 59, 79, 92, 95, 99, 106, 125, 131, 136, 145, 151, 154, 162, 172, 181, 184, 185, 193, 217, 222, 230, 231, 235, 245, 252, 264, 271, 287, 290, 291, 292, 311, 319, 321, 324, 325, 333, 354, 356, 382, 392, 400, 430, 459, 467, 473, 478, 493, 495, 511, 518, 528, 531, 541, 545, 552, 561, 571, 580, 586, 591, 609, 614, 620, 636, 638, 640, 646, 653, 663, 664, 693, 705, 726, 752, 754, 760, 770, 784, 793, 797, 830, 876, 889, 918, 925, 934, 951, 960, 973, 986, 989, 990, 993], 'values': [0.051331, 0.502887, 0.166458, 0.981803, 0.473514, 0.4653, 0.704692, 0.299539, 0.605007, 0.001821, 0.612247, 0.20294, 0.307676, 0.039077, 0.246199, 0.183483, 0.290873, 0.50727, 0.483406, 0.007947, 0.427872, 0.951975, 0.761097, 0.920098, 0.190474, 0.433741, 0.346492, 0.052065, 0.832253, 0.168165, 0.149203, 0.786062, 0.963168, 0.035258, 0.955501, 0.590641, 0.138426, 0.911361, 0.71722, 0.072034, 0.517648, 0.968779, 0.480174, 0.395554, 0.278859, 0.19239, 0.808107, 0.346126, 0.88346, 0.628155, 0.289531, 0.971333, 0.930341, 0.171938, 0.870009, 0.16918, 0.382264, 0.28136, 0.795893, 0.407454, 0.542961, 0.481764, 0.163725, 0.834179, 0.501071, 0.250244, 0.218221, 0.769889, 0.976098, 0.698947, 0.042155, 0.676616, 0.386043, 0.303742, 0.889136, 0.544955, 0.35605, 0.906034, 0.153586, 0.563263, 0.752793, 0.303353, 0.565097, 0.611277, 0.687791, 0.591125, 0.210591, 0.570756, 0.019015, 0.826023, 0.64314, 0.370911, 0.386581, 0.780488, 0.872206, 0.421009, 0.648609, 0.776962, 0.174285, 0.9962]}, '': [0.42187473, 0.5312091, 0.02644113, 0.4755205]}}, {'id': 600005, 'payload': {'city': 'Berlin'}, 'vector': {'': [0.81253946, 0.20593515, 0.124255456, 0.6036452], 'sparse-text': {'indices': [6, 25, 27, 40, 45, 52, 59, 74, 76, 85, 90, 98, 104, 109, 121, 152, 154, 156, 161, 163, 167, 180, 208, 219, 228, 257, 278, 281, 282, 285, 294, 296, 298, 307, 308, 310, 314, 318, 333, 352, 358, 387, 403, 426, 439, 454, 461, 465, 480, 496, 499, 501, 508, 525, 535, 537, 545, 548, 552, 566, 571, 572, 576, 579, 585, 590, 608, 613, 615, 617, 635, 644, 655, 658, 667, 682, 685, 713, 718, 746, 751, 768, 775, 802, 824, 825, 831, 834, 847, 850, 864, 867, 878, 894, 896, 920, 941, 949, 971, 984], 'values': [0.006099, 0.041014, 0.595597, 0.924165, 0.997881, 0.433743, 0.028598, 0.63686, 0.592393, 0.783584, 0.62471, 0.424795, 0.469653, 0.048767, 0.838689, 0.857366, 0.832501, 0.462223, 0.363119, 0.848415, 0.758556, 0.203476, 0.583074, 0.102271, 0.431838, 0.086634, 0.903478, 0.777956, 0.06073, 0.767535, 0.886442, 0.981458, 0.646013, 0.641629, 0.042603, 0.530816, 0.389091, 0.16402, 0.207283, 0.476958, 0.601659, 0.080345, 0.76917, 0.810417, 0.604566, 0.92758, 0.863258, 0.849647, 0.144949, 0.359851, 0.726493, 0.025558, 0.36306, 0.480009, 0.4135, 0.401027, 0.258937, 0.715953, 0.378162, 0.86779, 0.042396, 0.632647, 0.769226, 0.543659, 0.774324, 0.905226, 0.16748, 0.198984, 0.00918, 0.842955, 0.700729, 0.284272, 0.296666, 0.551427, 0.823913, 0.072204, 0.558331, 0.510899, 0.364274, 0.853089, 0.548604, 0.183164, 0.226698, 0.324472, 0.329742, 0.534086, 0.035003, 0.368365, 0.271417, 0.417472, 0.115758, 0.914352, 0.128239, 0.311827, 0.770231, 0.899918, 0.299926, 0.064528, 0.545202, 0.298211]}}}), ({'id': 600009, 'payload': {'city': 'London'}, 'vector': {'': [0.7894161, 0.667402, 0.025969604, 0.50153124], 'sparse-text': {'indices': [14, 32, 36, 41, 48, 61, 64, 75, 85, 86, 99, 126, 128, 144, 153, 156, 194, 199, 202, 205, 225, 229, 230, 240, 245, 263, 267, 271, 277, 293, 297, 314, 317, 321, 348, 358, 378, 379, 385, 390, 391, 405, 407, 433, 434, 438, 440, 442, 447, 467, 468, 478, 489, 490, 497, 498, 505, 513, 516, 517, 518, 525, 538, 541, 550, 572, 597, 606, 614, 615, 616, 617, 623, 665, 670, 671, 677, 687, 699, 702, 703, 760, 763, 800, 807, 818, 831, 832, 851, 859, 869, 877, 891, 909, 952, 955, 973, 975, 983, 988], 'values': [0.448212, 0.170333, 0.606896, 0.883359, 0.202761, 0.146883, 0.332051, 0.197149, 0.524481, 0.292195, 0.48661, 0.364031, 0.606812, 0.99962, 0.797364, 0.607985, 0.485762, 0.600984, 0.803045, 0.640609, 0.629687, 0.769065, 0.922118, 0.783779, 0.750982, 0.892472, 0.888338, 0.863265, 0.331559, 0.651809, 0.279892, 0.117069, 0.603923, 0.282886, 0.899831, 0.649253, 0.964528, 0.985153, 0.401254, 0.150782, 0.879783, 0.186821, 0.070318, 0.703261, 0.529993, 0.542175, 0.077145, 0.927767, 0.263565, 0.164087, 0.531894, 0.365703, 0.617613, 0.760771, 0.44059, 0.266756, 0.708925, 0.497577, 0.618418, 0.750256, 0.632786, 0.019753, 0.548016, 0.377672, 0.668653, 0.892326, 0.89184, 0.262217, 0.737175, 0.420688, 0.99379, 0.541679, 0.441208, 0.848622, 0.94375, 0.686847, 0.552683, 0.953006, 0.491247, 0.339792, 0.334259, 0.094798, 0.12486, 0.367828, 0.35783, 0.369286, 0.966436, 0.760865, 0.499926, 0.737742, 0.087413, 0.435169, 0.611862, 0.133252, 0.513992, 0.329088, 0.345209, 0.67223, 0.322557, 0.581136]}}}, {'id': 600006, 'payload': {'city': 'Berlin'}, 'vector': {'sparse-text': {'indices': [9, 35, 57, 60, 64, 74, 106, 139, 145, 157, 164, 176, 178, 214, 218, 242, 243, 249, 250, 264, 277, 292, 322, 329, 342, 376, 387, 401, 402, 411, 418, 422, 429, 459, 472, 476, 482, 486, 490, 491, 503, 504, 530, 534, 543, 546, 562, 565, 568, 576, 581, 587, 593, 602, 607, 609, 611, 613, 617, 631, 635, 654, 660, 666, 667, 674, 697, 706, 707, 717, 731, 733, 751, 767, 800, 803, 805, 808, 812, 815, 818, 820, 825, 836, 858, 862, 871, 879, 891, 906, 908, 912, 919, 921, 925, 939, 940, 942, 952, 998], 'values': [0.665641, 0.348418, 0.920225, 0.304783, 0.644, 0.244778, 0.768284, 0.308307, 0.368344, 0.95178, 0.744845, 0.831767, 0.419561, 0.944551, 0.831735, 0.360605, 0.343198, 0.064876, 0.605045, 0.818796, 0.018277, 0.333566, 0.572912, 0.262463, 0.630656, 0.484249, 0.260256, 0.57141, 0.762597, 0.272381, 0.02535, 0.267866, 0.14959, 0.10948, 0.676837, 0.999768, 0.573049, 0.82889, 0.498438, 0.155838, 0.240857, 0.217347, 0.673608, 0.545291, 0.781386, 0.252265, 0.043313, 0.570599, 0.628222, 0.518632, 0.751526, 0.233457, 0.136045, 0.995537, 0.802237, 0.326649, 0.988772, 0.471338, 0.627995, 0.420165, 0.757587, 0.366405, 0.215607, 0.519351, 0.721701, 0.046504, 0.73863, 0.298446, 0.333906, 0.369888, 0.683933, 0.727406, 0.233519, 0.648021, 0.081425, 0.651873, 0.421883, 0.464844, 0.755799, 0.052183, 0.777377, 0.038007, 0.221064, 0.964171, 0.599638, 0.839712, 0.255356, 0.703824, 0.68043, 0.699903, 0.406731, 0.142973, 0.311669, 0.483278, 0.163661, 0.146876, 0.212925, 0.330301, 0.774281, 0.770082]}, '': [0.05512371, 0.62330705, 0.43790078, 0.89942074]}}), ({'id': 600010, 'payload': {'city': 'Tokyo'}, 'vector': {'': [0.7763993, 0.7600899, 0.51723963, 0.83512384], 'sparse-text': {'indices': [7, 24, 27, 31, 34, 42, 63, 77, 86, 101, 103, 144, 165, 182, 197, 215, 239, 240, 244, 266, 273, 278, 284, 288, 294, 313, 323, 351, 352, 357, 358, 381, 391, 414, 418, 424, 450, 455, 459, 460, 461, 463, 464, 468, 478, 482, 484, 494, 510, 518, 527, 528, 545, 546, 548, 559, 561, 601, 614, 620, 629, 630, 638, 652, 654, 658, 660, 689, 696, 705, 725, 757, 776, 795, 800, 803, 804, 809, 833, 844, 846, 872, 891, 896, 902, 905, 906, 910, 919, 924, 932, 943, 953, 962, 963, 966, 967, 970, 984, 988], 'values': [0.829625, 0.649304, 0.913185, 0.879714, 0.645042, 0.909209, 0.20839, 0.211604, 0.829925, 0.786796, 0.287536, 0.156462, 0.896461, 0.696961, 0.962225, 0.629465, 0.601812, 0.120582, 0.283921, 0.134829, 0.938631, 0.221892, 0.494347, 0.222039, 0.32696, 0.127167, 0.398116, 0.116005, 0.194195, 0.787755, 0.584176, 0.604014, 0.91686, 0.6299, 0.254519, 0.175642, 0.196103, 0.496211, 0.719298, 0.059484, 0.22264, 0.10729, 0.565188, 0.894557, 0.956157, 0.619304, 0.82137, 0.875578, 0.94013, 0.880087, 0.058407, 0.198269, 0.992556, 0.346219, 0.619325, 0.33261, 0.297431, 0.542079, 0.863301, 0.437649, 0.265724, 0.41253, 0.300215, 0.633852, 0.263683, 0.307763, 0.929224, 0.219975, 0.047462, 0.830523, 0.510545, 0.253806, 0.944736, 0.350355, 0.285221, 0.353977, 0.091595, 0.357314, 0.888604, 0.318702, 0.653363, 0.147729, 0.131787, 0.04044, 0.786312, 0.192083, 0.625958, 0.389337, 0.581073, 0.549853, 0.028109, 0.138965, 0.64118, 0.031081, 0.970741, 0.06967, 0.770267, 0.219198, 0.613423, 0.282455]}}}, {'id': 600007, 'payload': {'city': 'London'}, 'vector': {'': [0.033625696, 0.94703394, 0.077850044, 0.58111423], 'sparse-text': {'indices': [9, 12, 21, 31, 35, 40, 62, 75, 84, 115, 119, 130, 139, 145, 148, 168, 193, 227, 230, 231, 238, 243, 253, 266, 269, 270, 286, 295, 297, 303, 314, 318, 321, 339, 349, 350, 355, 360, 367, 368, 378, 395, 398, 401, 411, 470, 473, 475, 478, 524, 526, 545, 546, 551, 559, 561, 563, 575, 580, 587, 590, 599, 601, 619, 642, 649, 650, 651, 653, 672, 675, 683, 697, 704, 715, 732, 736, 743, 745, 746, 771, 774, 778, 784, 785, 786, 792, 809, 821, 822, 859, 915, 927, 953, 969, 970, 974, 978, 984, 994], 'values': [0.917857, 0.982746, 0.78812, 0.903924, 0.358587, 0.760152, 0.133249, 0.14833, 0.220425, 0.075314, 0.293215, 0.965109, 0.42874, 0.087525, 0.317281, 0.327753, 0.804944, 0.029775, 0.118523, 0.714002, 0.390026, 0.37008, 0.016912, 0.823931, 0.255645, 0.536204, 0.136131, 0.558526, 0.615286, 0.793494, 0.449356, 0.533933, 0.415334, 0.11821, 0.052219, 0.180582, 0.914738, 0.96509, 0.012821, 0.681724, 0.872383, 0.281288, 0.807585, 0.183325, 0.467274, 0.13346, 0.371189, 0.439168, 0.18781, 0.545961, 0.128614, 0.916686, 0.805591, 0.6815, 0.267806, 0.03314, 0.500474, 0.300532, 0.707826, 0.229606, 0.279991, 0.036378, 0.900211, 0.676212, 0.257466, 0.609909, 0.938757, 0.976938, 0.344318, 0.842274, 0.421745, 0.969104, 0.062404, 0.62196, 0.798657, 0.990517, 0.665366, 0.562486, 0.84874, 0.137503, 0.803548, 0.195051, 0.658588, 0.025601, 0.233617, 0.86609, 0.67359, 0.025783, 0.716486, 0.700585, 0.564038, 0.500688, 0.390762, 0.98823, 0.925078, 0.045717, 0.538811, 0.72606, 0.235757, 0.573134]}}}), ({'id': 600011, 'payload': {'city': 'Berlin'}, 'vector': {'sparse-text': {'indices': [22, 24, 26, 27, 48, 54, 71, 80, 89, 101, 105, 124, 133, 137, 141, 142, 157, 161, 188, 204, 206, 209, 224, 233, 249, 253, 265, 279, 292, 300, 306, 318, 326, 336, 350, 356, 363, 370, 388, 392, 403, 405, 425, 429, 430, 462, 464, 474, 482, 487, 492, 500, 556, 560, 572, 585, 587, 604, 614, 616, 621, 630, 633, 649, 652, 660, 694, 695, 702, 721, 725, 727, 733, 741, 743, 763, 767, 769, 770, 776, 786, 788, 789, 804, 824, 837, 853, 867, 876, 886, 887, 911, 915, 919, 924, 931, 946, 981, 998, 999], 'values': [0.626455, 0.930807, 0.155153, 0.595311, 0.957727, 0.206947, 0.440228, 0.965955, 0.638461, 0.116691, 0.813608, 0.612273, 0.305532, 0.413685, 0.515888, 0.998804, 0.662264, 0.838935, 0.348252, 0.40849, 0.408926, 0.935134, 0.083396, 0.395725, 0.323421, 0.695196, 0.344808, 0.453154, 0.933667, 0.496032, 0.838563, 0.75508, 0.139507, 0.397289, 0.452625, 0.776402, 0.906087, 0.20355, 0.496707, 0.316035, 0.209783, 0.917537, 0.435556, 0.834406, 0.872285, 0.35291, 0.398447, 0.88373, 0.954994, 0.241685, 0.151968, 0.610799, 0.602435, 0.488695, 0.497835, 0.151111, 0.336866, 0.090644, 0.079094, 0.588042, 0.596095, 0.728127, 0.670348, 0.583897, 0.29814, 0.304064, 0.433678, 0.932278, 0.710503, 0.325287, 0.193582, 0.66622, 0.779906, 0.606336, 0.276474, 0.513945, 0.53911, 0.119407, 0.170619, 0.851701, 0.065036, 0.181035, 0.499823, 0.28528, 0.7181, 0.78441, 0.175406, 0.744858, 0.614128, 0.000995, 0.595657, 0.833265, 0.675551, 0.890573, 0.943534, 0.844827, 0.576888, 0.961289, 0.580374, 0.954699]}, '': [0.76549983, 0.17991227, 0.35764322, 0.8976913]}}, {'id': 600008, 'payload': {'city': 'Tokyo'}, 'vector': {'sparse-text': {'indices': [9, 10, 28, 33, 39, 54, 59, 79, 92, 95, 99, 106, 125, 131, 136, 145, 151, 154, 162, 172, 181, 184, 185, 193, 217, 222, 230, 231, 235, 245, 252, 264, 271, 287, 290, 291, 292, 311, 319, 321, 324, 325, 333, 354, 356, 382, 392, 400, 430, 459, 467, 473, 478, 493, 495, 511, 518, 528, 531, 541, 545, 552, 561, 571, 580, 586, 591, 609, 614, 620, 636, 638, 640, 646, 653, 663, 664, 693, 705, 726, 752, 754, 760, 770, 784, 793, 797, 830, 876, 889, 918, 925, 934, 951, 960, 973, 986, 989, 990, 993], 'values': [0.051331, 0.502887, 0.166458, 0.981803, 0.473514, 0.4653, 0.704692, 0.299539, 0.605007, 0.001821, 0.612247, 0.20294, 0.307676, 0.039077, 0.246199, 0.183483, 0.290873, 0.50727, 0.483406, 0.007947, 0.427872, 0.951975, 0.761097, 0.920098, 0.190474, 0.433741, 0.346492, 0.052065, 0.832253, 0.168165, 0.149203, 0.786062, 0.963168, 0.035258, 0.955501, 0.590641, 0.138426, 0.911361, 0.71722, 0.072034, 0.517648, 0.968779, 0.480174, 0.395554, 0.278859, 0.19239, 0.808107, 0.346126, 0.88346, 0.628155, 0.289531, 0.971333, 0.930341, 0.171938, 0.870009, 0.16918, 0.382264, 0.28136, 0.795893, 0.407454, 0.542961, 0.481764, 0.163725, 0.834179, 0.501071, 0.250244, 0.218221, 0.769889, 0.976098, 0.698947, 0.042155, 0.676616, 0.386043, 0.303742, 0.889136, 0.544955, 0.35605, 0.906034, 0.153586, 0.563263, 0.752793, 0.303353, 0.565097, 0.611277, 0.687791, 0.591125, 0.210591, 0.570756, 0.019015, 0.826023, 0.64314, 0.370911, 0.386581, 0.780488, 0.872206, 0.421009, 0.648609, 0.776962, 0.174285, 0.9962]}, '': [0.42187473, 0.5312091, 0.02644113, 0.4755205]}})]"
--------------------------- Captured stdout teardown ---------------------------
Killing 44159
Killing 44189
Killing 44190
Killing 44427
Killing 44428
=============================== warnings summary ===============================
tests/.venv/lib/python3.13/site-packages/jwt/api_jwt.py:153
consensus_tests/auth_tests/test_jwt_validation.py::test_value_exists_claim
consensus_tests/auth_tests/test_jwt_validation.py::test_recommend_collection_lookup
/home/runner/work/qdrant/qdrant/tests/.venv/lib/python3.13/site-packages/jwt/api_jwt.py:153: InsecureKeyLengthWarning: The HMAC key is 17 bytes long, which is below the minimum recommended length of 32 bytes for SHA256. See RFC 7518 Section 3.2.
return self._jws.encode(
consensus_tests/auth_tests/test_jwt_validation.py::test_value_exists_claim
/home/runner/work/qdrant/qdrant/tests/.venv/lib/python3.13/site-packages/jwt/api_jwt.py:153: InsecureKeyLengthWarning: The HMAC key is 25 bytes long, which is below the minimum recommended length of 32 bytes for SHA256. See RFC 7518 Section 3.2.
return self._jws.encode(
consensus_tests/test_cluster_operation_coalescing.py: 2 warnings
consensus_tests/test_read_consistency.py: 3 warnings
consensus_tests/test_replicate_points.py: 2 warnings
consensus_tests/test_shard_consistency.py: 3 warnings
consensus_tests/test_shard_snapshot_transfer.py: 6 warnings
consensus_tests/test_shard_snapshot_transfer_cancel.py: 3 warnings
consensus_tests/test_shard_stream_transfer.py: 9 warnings
consensus_tests/test_shard_wal_delta_transfer.py: 12 warnings
consensus_tests/test_snapshot_consistency.py: 3 warnings
consensus_tests/test_tenant_promotion.py: 2 warnings
consensus_tests/test_triple_replication.py: 1 warning
consensus_tests/test_two_follower_nodes_down.py: 1 warning
/opt/hostedtoolcache/Python/3.13.12/x64/lib/python3.13/multiprocessing/popen_fork.py:67: DeprecationWarning: This process (pid=27358) is multi-threaded, use of fork() may lead to deadlocks in the child.
self.pid = os.fork()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
============================= slowest 10 durations =============================
56.83s call consensus_tests/test_partial_snapshot.py::test_partial_snapshot_read_lock
56.48s call consensus_tests/test_partial_snapshot.py::test_partial_snapshot_recovery_lock[True]
55.60s call consensus_tests/test_partial_snapshot.py::test_partial_snapshot_recovery_lock[False]
37.85s call consensus_tests/test_collection_creation_after_dropping.py::test_collection_creation_after_dropping
30.48s call consensus_tests/test_snapshot_backpressure_freeze.py::test_snapshot_backpressure_freeze
30.04s call consensus_tests/test_streaming_snapshot_consensus_freeze.py::test_streaming_snapshot_freezes_collection_via_consensus
26.43s call consensus_tests/test_snapshot_consistency.py::test_shard_wal_delta_transfer_manual_recovery
23.92s call consensus_tests/test_shard_wal_delta_transfer.py::test_shard_fallback_on_big_diff
23.54s call consensus_tests/test_cluster_rejoin.py::test_reject_rejoin_cluster_same_uri[False]
23.54s call consensus_tests/test_cluster_rejoin.py::test_reject_rejoin_cluster_same_uri[True]
=========================== short test summary info ============================
FAILED tests/consensus_tests/test_shard_wal_delta_transfer.py::test_shard_wal_delta_transfer_manual_recovery_chain - AssertionError: Data on all nodes should be consistent
assert False
FAILED tests/consensus_tests/test_shard_wal_delta_transfer.py::test_shard_wal_delta_transfer_manual_recovery_chain - AssertionError: Data on all nodes should be consistenthttps://github.com/qdrant/qdrant/actions/runs/22872178777/job/66354498688?pr=8335
consensus-test-logs.zip