Skip to content

Backup_Restore_concurrency_check_node #47216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vitlibar merged 4 commits into from
Mar 14, 2023
Merged

Backup_Restore_concurrency_check_node #47216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d2dbd5f
Updated to use tryGet instead of get for checking stage of backups/re…
SmitaRKulkarni Mar 3, 2023
cebe323
Automatic style fix
robot-clickhouse Mar 3, 2023
9a35a43
Merge branch 'master' into Backup_Restore_concurrency_check_node
SmitaRKulkarni Mar 13, 2023
f76b97c
Set num of nodes to 4 for backup restore disallow concurrency test
SmitaRKulkarni Mar 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions src/Backups/BackupCoordinationRemote.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -750,9 +750,12 @@ bool BackupCoordinationRemote::hasConcurrentBackups(const std::atomic<size_t> &)
if (existing_backup_uuid == toString(backup_uuid))
continue;

const auto status = zk->get(root_zookeeper_path + "/" + existing_backup_path + "/stage");
if (status != Stage::COMPLETED)
return true;
String status;
if (zk->tryGet(root_zookeeper_path + "/" + existing_backup_path + "/stage", status))
{
if (status != Stage::COMPLETED)
return true;
}
}

zk->createIfNotExists(backup_stage_path, "");
Expand Down
31 changes: 16 additions & 15 deletions tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@

cluster = ClickHouseCluster(__file__)

num_nodes = 10
num_nodes = 4
ddl_task_timeout = 640


def generate_cluster_def():
Expand Down Expand Up @@ -85,7 +86,7 @@ def drop_after_test():
node0.query(
"DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY",
settings={
"distributed_ddl_task_timeout": 360,
"distributed_ddl_task_timeout": ddl_task_timeout,
},
)

Expand All @@ -107,7 +108,7 @@ def create_and_fill_table():
"ORDER BY x"
)
for i in range(num_nodes):
nodes[i].query(f"INSERT INTO tbl SELECT number FROM numbers(40000000)")
nodes[i].query(f"INSERT INTO tbl SELECT number FROM numbers(80000000)")


# All the tests have concurrent backup/restores with same backup names
Expand Down Expand Up @@ -145,7 +146,7 @@ def test_concurrent_backups_on_same_node():
nodes[0].query(
f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY",
settings={
"distributed_ddl_task_timeout": 360,
"distributed_ddl_task_timeout": ddl_task_timeout,
},
)
nodes[0].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}")
Expand Down Expand Up @@ -202,7 +203,7 @@ def test_concurrent_restores_on_same_node():
nodes[0].query(
f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY",
settings={
"distributed_ddl_task_timeout": 360,
"distributed_ddl_task_timeout": ddl_task_timeout,
},
)
restore_id = (
Expand All @@ -226,44 +227,44 @@ def test_concurrent_restores_on_different_node():
backup_name = new_backup_name()

id = (
nodes[0]
nodes[1]
.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name} ASYNC")
.split("\t")[0]
)
assert_eq_with_retry(
nodes[0],
nodes[1],
f"SELECT status FROM system.backups WHERE status == 'CREATING_BACKUP' AND id = '{id}'",
"CREATING_BACKUP",
)

assert_eq_with_retry(
nodes[0],
nodes[1],
f"SELECT status FROM system.backups WHERE status == 'BACKUP_CREATED' AND id = '{id}'",
"BACKUP_CREATED",
)

nodes[0].query(
nodes[1].query(
f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY",
settings={
"distributed_ddl_task_timeout": 360,
"distributed_ddl_task_timeout": ddl_task_timeout,
},
)
restore_id = (
nodes[0]
nodes[1]
.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC")
.split("\t")[0]
)
assert_eq_with_retry(
nodes[0],
f"SELECT status FROM system.backups WHERE status == 'RESTORING'",
nodes[1],
f"SELECT status FROM system.backups WHERE status == 'RESTORING' AND id == '{restore_id}'",
"RESTORING",
)
assert "Concurrent restores not supported" in nodes[1].query_and_get_error(
assert "Concurrent restores not supported" in nodes[0].query_and_get_error(
f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}"
)

assert_eq_with_retry(
nodes[0],
nodes[1],
f"SELECT status FROM system.backups WHERE status == 'RESTORED' AND id == '{restore_id}'",
"RESTORED",
)