Skip to content

Commit 5579d9f

Browse files
robot-clickhouserobot-clickhouse
committed
Backport #69563 to 24.9: Improve restoring of access entities' dependencies #2 
1 parent 1f7e969 commit 5579d9f

File tree

60 files changed

+1199
-468
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1199
-468
lines changed

‎src/Access/AccessBackup.cpp‎

Lines changed: 471 additions & 220 deletions
Large diffs are not rendered by default.

‎src/Access/AccessBackup.h‎

Lines changed: 98 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
#pragma once
22

3+
#include <Common/Logger.h>
34
#include <Core/UUID.h>
45
#include <unordered_map>
5-
#include <unordered_set>
66

77

88
namespace DB
@@ -12,6 +12,7 @@ enum class AccessEntityType : uint8_t;
1212
struct IAccessEntity;
1313
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
1414
class AccessRightsElements;
15+
class IAccessStorage;
1516
class IBackup;
1617
using BackupPtr = std::shared_ptr<const IBackup>;
1718
class IBackupEntry;
@@ -20,37 +21,118 @@ struct RestoreSettings;
2021
enum class RestoreAccessCreationMode : uint8_t;
2122

2223

23-
/// Makes a backup of access entities of a specified type.
24-
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
25-
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
26-
const String & data_path_in_backup,
27-
size_t counter,
28-
const AccessControl & access_control);
24+
/// Makes a backup entry for of a set of access entities.
25+
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
26+
const std::vector<UUID> & entities_ids,
27+
const std::unordered_map<UUID, AccessEntityPtr> & all_entities,
28+
bool write_dependents,
29+
const String & data_path_in_backup);
2930

31+
struct AccessEntitiesToRestore
32+
{
33+
/// Access entities loaded from backup with new randomly generated UUIDs.
34+
std::vector<std::pair<UUID /* new_id */, AccessEntityPtr /* new_entity */>> new_entities;
35+
36+
/// Dependents are access entities which exist already and they should be updated after restoring.
37+
/// For example, if there were a role granted to a user: `CREATE USER user1; CREATE ROLE role1; GRANT role1 TO user1`,
38+
/// and we're restoring only role `role1` because user `user1` already exists,
39+
/// then user `user1` should be modified after restoring role `role1` to add this grant `GRANT role1 TO user1`.
40+
struct Dependent
41+
{
42+
/// UUID of an existing access entities.
43+
UUID existing_id;
44+
45+
/// Source access entity from backup to copy dependencies from.
46+
AccessEntityPtr source;
47+
};
48+
using Dependents = std::vector<Dependent>;
49+
Dependents dependents;
50+
};
3051

3152
/// Restores access entities from a backup.
53+
void restoreAccessEntitiesFromBackup(
54+
IAccessStorage & access_storage,
55+
const AccessEntitiesToRestore & entities_to_restore,
56+
const RestoreSettings & restore_settings);
57+
58+
59+
/// Loads access entities from a backup and prepares them for insertion into an access storage.
3260
class AccessRestorerFromBackup
3361
{
3462
public:
3563
AccessRestorerFromBackup(const BackupPtr & backup_, const RestoreSettings & restore_settings_);
3664
~AccessRestorerFromBackup();
3765

3866
/// Adds a data path to loads access entities from.
39-
void addDataPath(const String & data_path);
67+
void addDataPath(const String & data_path_in_backup);
68+
69+
/// Loads access entities from the backup.
70+
void loadFromBackup();
4071

4172
/// Checks that the current user can do restoring.
73+
/// Function loadFromBackup() must be called before that.
4274
AccessRightsElements getRequiredAccess() const;
4375

44-
/// Inserts all access entities loaded from all the paths added by addDataPath().
45-
std::vector<std::pair<UUID, AccessEntityPtr>> getAccessEntities(const AccessControl & access_control) const;
76+
/// Generates random IDs for access entities we're restoring to insert them into an access storage;
77+
/// and finds IDs of existing access entities which are used as dependencies.
78+
void generateRandomIDsAndResolveDependencies(const AccessControl & access_control);
79+
80+
/// Returns access entities loaded from backup and prepared for insertion into an access storage.
81+
/// Both functions loadFromBackup() and generateRandomIDsAndResolveDependencies() must be called before that.
82+
AccessEntitiesToRestore getEntitiesToRestore(const String & data_path_in_backup) const;
4683

4784
private:
48-
BackupPtr backup;
49-
RestoreAccessCreationMode creation_mode;
50-
bool allow_unresolved_dependencies = false;
51-
std::vector<std::pair<UUID, AccessEntityPtr>> entities;
52-
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
53-
std::unordered_set<String> data_paths;
85+
const BackupPtr backup;
86+
const RestoreAccessCreationMode creation_mode;
87+
const bool skip_unresolved_dependencies;
88+
const bool update_dependents;
89+
const LoggerPtr log;
90+
91+
/// Whether loadFromBackup() finished.
92+
bool loaded = false;
93+
94+
/// Whether generateRandomIDsAndResolveDependencies() finished.
95+
bool ids_assigned = false;
96+
97+
Strings data_paths_in_backup;
98+
String data_path_with_entities_to_restore;
99+
100+
/// Information about an access entity loaded from the backup.
101+
struct EntityInfo
102+
{
103+
UUID id;
104+
String name;
105+
AccessEntityType type;
106+
107+
AccessEntityPtr entity = nullptr; /// Can be nullptr if `restore=false`.
108+
109+
/// Index in `data_paths_in_backup`.
110+
size_t data_path_index = 0;
111+
112+
/// Whether we're going to restore this entity.
113+
/// For example,
114+
/// in case of `RESTORE TABLE system.roles` this flag is true for all the roles loaded from the backup, and
115+
/// in case of `RESTORE ALL` this flag is always true.
116+
bool restore = false;
117+
118+
/// Whether this entity info was added as a dependency of another entity which we're going to restore.
119+
/// For example, if we're going to restore the following user: `CREATE USER user1 DEFAULT ROLE role1, role2 SETTINGS PROFILE profile1, profile2`
120+
/// then `restore=true` for `user1` and `is_dependency=true` for `role1`, `role2`, `profile1`, `profile2`.
121+
/// Flags `restore` and `is_dependency` both can be set at the same time.
122+
bool is_dependency = false;
123+
124+
/// Whether this entity info is a dependent of another entity which we're going to restore.
125+
/// For example, if we're going to restore role `role1` and there is also the following user stored in the backup:
126+
/// `CREATE USER user1 DEFAULT ROLE role1`, then `is_dependent=true` for `user1`.
127+
/// This flags is set by generateRandomIDsAndResolveDependencies().
128+
bool is_dependent = false;
129+
130+
/// New UUID for this entity - either randomly generated or copied from an existing entity.
131+
/// This UUID is assigned by generateRandomIDsAndResolveDependencies().
132+
std::optional<UUID> new_id = std::nullopt;
133+
};
134+
135+
std::unordered_map<UUID, EntityInfo> entity_infos;
54136
};
55137

56138
}

‎src/Access/AccessControl.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -629,9 +629,9 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
629629
}
630630
}
631631

632-
void AccessControl::restoreFromBackup(RestorerFromBackup & restorer)
632+
void AccessControl::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
633633
{
634-
MultipleAccessStorage::restoreFromBackup(restorer);
634+
MultipleAccessStorage::restoreFromBackup(restorer, data_path_in_backup);
635635
changes_notifier->sendNotifications();
636636
}
637637

‎src/Access/AccessControl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ class AccessControl : public MultipleAccessStorage
124124
AuthResult authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const String & forwarded_address) const;
125125

126126
/// Makes a backup of access entities.
127-
void restoreFromBackup(RestorerFromBackup & restorer) override;
127+
void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override;
128128

129129
void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
130130

‎src/Access/DiskAccessStorage.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -676,7 +676,7 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_
676676
if (!entry.entity)
677677
entry.entity = readAccessEntityFromDisk(id);
678678
auto old_entity = entry.entity;
679-
auto new_entity = update_func(old_entity);
679+
auto new_entity = update_func(old_entity, id);
680680

681681
if (!new_entity->isTypeOf(old_entity->getType()))
682682
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());

‎src/Access/GrantedRoles.cpp‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,16 @@ std::vector<UUID> GrantedRoles::findDependencies() const
176176
return res;
177177
}
178178

179+
bool GrantedRoles::hasDependencies(const std::unordered_set<UUID> & ids) const
180+
{
181+
for (const auto & role_id : roles)
182+
{
183+
if (ids.contains(role_id))
184+
return true;
185+
}
186+
return false;
187+
}
188+
179189
void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
180190
{
181191
std::vector<UUID> new_ids;
@@ -221,4 +231,56 @@ void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & ol
221231
}
222232
}
223233

234+
void GrantedRoles::copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids)
235+
{
236+
bool found = false;
237+
238+
for (const auto & role_id : src.roles)
239+
{
240+
if (ids.contains(role_id))
241+
{
242+
roles.emplace(role_id);
243+
found = true;
244+
}
245+
}
246+
247+
if (found)
248+
{
249+
for (const auto & role_id : src.roles_with_admin_option)
250+
{
251+
if (ids.contains(role_id))
252+
roles_with_admin_option.emplace(role_id);
253+
}
254+
}
255+
}
256+
257+
void GrantedRoles::removeDependencies(const std::unordered_set<UUID> & ids)
258+
{
259+
bool found = false;
260+
261+
for (auto it = roles.begin(); it != roles.end();)
262+
{
263+
if (ids.contains(*it))
264+
{
265+
it = roles.erase(it);
266+
found = true;
267+
}
268+
else
269+
{
270+
++it;
271+
}
272+
}
273+
274+
if (found)
275+
{
276+
for (auto it = roles_with_admin_option.begin(); it != roles_with_admin_option.end();)
277+
{
278+
if (ids.contains(*it))
279+
it = roles_with_admin_option.erase(it);
280+
else
281+
++it;
282+
}
283+
}
284+
}
285+
224286
}

‎src/Access/GrantedRoles.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,10 @@ class GrantedRoles
5858
friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); }
5959

6060
std::vector<UUID> findDependencies() const;
61+
bool hasDependencies(const std::unordered_set<UUID> & ids) const;
6162
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
63+
void copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids);
64+
void removeDependencies(const std::unordered_set<UUID> & ids);
6265

6366
private:
6467
boost::container::flat_set<UUID> roles;

‎src/Access/IAccessEntity.cpp‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,28 +9,4 @@ bool IAccessEntity::equal(const IAccessEntity & other) const
99
return (name == other.name) && (getType() == other.getType());
1010
}
1111

12-
void IAccessEntity::replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids)
13-
{
14-
if (old_to_new_ids.empty())
15-
return;
16-
17-
bool need_replace_dependencies = false;
18-
auto dependencies = entity->findDependencies();
19-
for (const auto & dependency : dependencies)
20-
{
21-
if (old_to_new_ids.contains(dependency))
22-
{
23-
need_replace_dependencies = true;
24-
break;
25-
}
26-
}
27-
28-
if (!need_replace_dependencies)
29-
return;
30-
31-
auto new_entity = entity->clone();
32-
new_entity->replaceDependencies(old_to_new_ids);
33-
entity = new_entity;
34-
}
35-
3612
}

‎src/Access/IAccessEntity.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,13 @@ struct IAccessEntity
4848

4949
/// Finds all dependencies.
5050
virtual std::vector<UUID> findDependencies() const { return {}; }
51+
virtual bool hasDependencies(const std::unordered_set<UUID> & /* ids */) const { return false; }
5152

5253
/// Replaces dependencies according to a specified map.
53-
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); }
54-
static void replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids);
54+
virtual void replaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
55+
virtual void copyDependenciesFrom(const IAccessEntity & /* src */, const std::unordered_set<UUID> & /* ids */) {}
56+
virtual void removeDependencies(const std::unordered_set<UUID> & /* ids */) {}
57+
virtual void clearAllExceptDependencies() {}
5558

5659
/// Whether this access entity should be written to a backup.
5760
virtual bool isBackupAllowed() const { return false; }
@@ -67,8 +70,6 @@ struct IAccessEntity
6770
{
6871
return std::make_shared<EntityClassT>(typeid_cast<const EntityClassT &>(*this));
6972
}
70-
71-
virtual void doReplaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
7273
};
7374

7475
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;

0 commit comments

Comments
 (0)