Skip to content

Commit e345898

Browse files
vinay-satishvinay-satish
committed
updating the cleanup_bifrost.rb
Signed-off-by: Vinay Satish <[email protected]>
1 parent 48cc345 commit e345898

File tree

1 file changed

+101
-19
lines changed

1 file changed

+101
-19
lines changed

‎src/chef-server-ctl/plugins/cleanup_bifrost.rb‎

Lines changed: 101 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,41 @@
22

33
CREATE_SQL = <<SQL
44
BEGIN;
5+
56
CREATE UNLOGGED TABLE IF NOT EXISTS cleanup_tracking_auth_actors(
67
authz_id CHAR(32)
78
);
9+
CREATE UNLOGGED TABLE IF NOT EXISTS cleanup_tracking_auth_objects(
10+
authz_id CHAR(32)
11+
);
812
913
CREATE OR REPLACE RULE cleanup_auth_actor_creation_tracking AS ON INSERT TO auth_actor DO INSERT INTO cleanup_tracking_auth_actors VALUES (NEW.authz_id);
14+
CREATE OR REPLACE RULE cleanup_auth_object_creation_tracking AS ON INSERT TO auth_object DO INSERT INTO cleanup_tracking_auth_objects VALUES (NEW.authz_id);
15+
1016
COMMIT;
1117
SQL
1218

1319
CREATE_SQL2 = <<SQL
20+
BEGIN;
21+
1422
CREATE UNLOGGED TABLE IF NOT EXISTS cleanup_known_auth_actors(
1523
authz_id CHAR(32)
1624
);
25+
CREATE UNLOGGED TABLE IF NOT EXISTS cleanup_known_auth_objects(
26+
authz_id CHAR(32)
27+
);
28+
29+
COMMIT;
1730
SQL
1831

1932
DELETE_SQL = <<SQL
2033
BEGIN;
2134
DROP RULE IF EXISTS cleanup_auth_actor_creation_tracking ON auth_actor;
2235
DROP TABLE IF EXISTS cleanup_tracking_auth_actors;
2336
DROP TABLE IF EXISTS cleanup_known_auth_actors;
37+
DROP RULE IF EXISTS cleanup_auth_object_creation_tracking ON auth_object;
38+
DROP TABLE IF EXISTS cleanup_tracking_auth_objects;
39+
DROP TABLE IF EXISTS cleanup_known_auth_objects;
2440
COMMIT;
2541
SQL
2642

@@ -38,13 +54,28 @@
3854
DELETE FROM auth_actor WHERE authz_id IN (SELECT authz_id FROM orphaned_auth_actors)
3955
SQL
4056

57+
CLEANUP_SQL2 = <<SQL
58+
WITH good_auth_objects AS (
59+
SELECT authz_id FROM cleanup_tracking_auth_objects
60+
UNION
61+
SELECT authz_id FROM cleanup_known_auth_objects
62+
),
63+
orphaned_auth_objects AS (
64+
SELECT authz_id
65+
FROM auth_object
66+
WHERE authz_id NOT IN (SELECT authz_id FROM good_auth_objects)
67+
ORDER BY id LIMIT $1
68+
)
69+
DELETE FROM auth_object WHERE authz_id IN (SELECT authz_id FROM orphaned_auth_objects)
70+
SQL
71+
4172
add_command_under_category "cleanup-bifrost", "cleanup", "Cleanup orphaned bifrost objects.", 2 do
4273
cleanup_args = ARGV[1..-1]
4374
options = {}
4475

4576
OptionParser.new do |opts|
4677
opts.banner = "#{ChefUtils::Dist::Server::SERVER_CTL} cleanup-bifrost [options]"
47-
opts.on("-b SIZE", "--batch-size SIZE", "How many authz actors to delete at a time") do |b|
78+
opts.on("-b SIZE", "--batch-size SIZE", "How many authz actors to delete at a time (default: 10000)") do |b|
4879
options[:batch_size] = b.to_i
4980
end
5081

@@ -98,7 +129,7 @@
98129
end
99130

100131
if options[:estimate_only]
101-
print_and_return_estimate(known_actor_list, bifrost_db)
132+
print_and_return_estimate(known_actor_list, known_object_list, bifrost_db)
102133
exit(0)
103134
end
104135

@@ -112,14 +143,14 @@ def run_cleanup(bifrost_db, batch_size, wait_time)
112143
begin
113144
puts "Sleeping #{wait_time} seconds to account for in-flight requests not captured by tracking table"
114145
sleep wait_time
115-
estimated_deletion_count = print_and_return_estimate(known_actor_list, bifrost_db)
146+
estimated_deletion_count = print_and_return_estimate(known_actor_list, known_object_list, bifrost_db)
116147

117148
if estimated_deletion_count <= 0
118149
puts "Estimated deletion count 0. Aborting"
119150
exit(0)
120151
end
121152

122-
install_known_actor_table(known_actor_list, bifrost_db)
153+
install_known_actor_object_table(known_actor_list, known_object_list, bifrost_db)
123154
run_bifrost_scan(batch_size, bifrost_db)
124155
ensure
125156
remove_bifrost_tracking_table(bifrost_db)
@@ -135,6 +166,23 @@ def known_actor_list
135166
end
136167
end
137168

169+
def known_object_list
170+
@known_object_list ||= timed "Fetching initial opscode_chef objects list" do
171+
objects = erchef_db.exec("SELECT authz_id FROM cookbook_artifacts
172+
UNION SELECT authz_id FROM cookbooks
173+
UNION SELECT authz_id FROM data_bags
174+
UNION SELECT authz_id FROM environments
175+
UNION SELECT authz_id FROM nodes
176+
UNION SELECT authz_id FROM orgs
177+
UNION SELECT authz_id FROM policies
178+
UNION SELECT authz_id FROM policy_groups
179+
UNION SELECT authz_id FROM roles")
180+
objects.map do |object|
181+
object["authz_id"]
182+
end
183+
end
184+
end
185+
138186
def safety_check(db)
139187
res = db.exec("SELECT * FROM pg_tables
140188
WHERE tablename='cleanup_tracking_auth_actors'")
@@ -143,6 +191,14 @@ def safety_check(db)
143191
puts "ERROR: If you are sure cleanup-bifrost is not running, you can clean up the tracking tables with: #{ChefUtils::Dist::Server::SERVER_CTL} cleanup-bifrost --force-cleanup"
144192
exit(1)
145193
end
194+
195+
res = db.exec("SELECT * FROM pg_tables
196+
WHERE tablename='cleanup_tracking_auth_objects'")
197+
if res.ntuples > 0
198+
puts "ERROR: cleanup_tracking_auth_objects already exists. cleanup-bifrost may be running."
199+
puts "ERROR: If you are sure cleanup-bifrost is not running, you can clean up the tracking tables with: #{ChefUtils::Dist::Server::SERVER_CTL} cleanup-bifrost --force-cleanup"
200+
exit(1)
201+
end
146202
end
147203

148204
def install_bifrost_tracking_table(db)
@@ -151,11 +207,16 @@ def install_bifrost_tracking_table(db)
151207
end
152208
end
153209

154-
def install_known_actor_table(list, db)
210+
def install_known_actor_object_table(actor_list, object_list, db)
155211
timed "Populating known actor table" do
156212
db.exec(CREATE_SQL2)
157213
db.copy_data("COPY cleanup_known_auth_actors FROM STDIN") do
158-
list.each do |id|
214+
actor_list.each do |id|
215+
db.put_copy_data(id.concat("\n"))
216+
end
217+
end
218+
db.copy_data("COPY cleanup_known_auth_objects FROM STDIN") do
219+
object_list.each do |id|
159220
db.put_copy_data(id.concat("\n"))
160221
end
161222
end
@@ -168,37 +229,58 @@ def remove_bifrost_tracking_table(db)
168229
end
169230
end
170231

171-
def fetch_auth_actor_count(db)
172-
timed "Fetching count from bifrost auth_actor" do
173-
db.exec("SELECT count(*) FROM auth_actor").first["count"].to_i
232+
def fetch_auth_count(db, type)
233+
timed "Fetching count from bifrost auth_" + type do
234+
db.exec("SELECT count(*) FROM auth_" + type).first["count"].to_i
174235
end
175236
end
176237

177-
def print_and_return_estimate(known_actor_list, db)
178-
tcount = fetch_auth_actor_count(db)
179-
estimated_del_count = [0, tcount - known_actor_list.length].max
238+
def print_and_return_estimate(known_actor_list, known_object_list, db)
239+
total_actors_count = fetch_auth_count(db, "actor")
240+
estimated_actors_del_count = [0, total_actors_count - known_actor_list.length].max
180241
puts "\n----------------------------------------"
181242
puts " Total #{ChefUtils::Dist::Infra::SHORT} users+clients: #{known_actor_list.length}"
182-
puts "Total bifrost auth_actors: #{tcount}"
183-
puts "Deletion Candidates (est): #{estimated_del_count}"
243+
puts "Total bifrost auth_actors: #{total_actors_count}"
244+
puts "Deletion Candidates (est): #{estimated_actors_del_count}"
245+
puts "----------------------------------------\n"
246+
247+
total_objects_count = fetch_auth_count(db, "object")
248+
estimated_objects_del_count = [0, total_objects_count - known_object_list.length].max
249+
puts "\n----------------------------------------"
250+
puts " Total #{ChefUtils::Dist::Infra::SHORT} objects: #{known_object_list.length}"
251+
puts "Total bifrost auth_objects: #{total_objects_count}"
252+
puts "Deletion Candidates (est): #{estimated_objects_del_count}"
184253
puts "----------------------------------------\n"
185254

186-
estimated_del_count
255+
estimated_actors_del_count + estimated_objects_del_count
187256
end
188257

189258
def run_bifrost_scan(batch_size, db)
190-
total_deleted = 0
259+
total_actors_deleted = 0
191260
loop do
192261
deletion_count = timed "Processing batch of #{batch_size} unknown auth_actors. " do
193262
count = db.exec(CLEANUP_SQL, [batch_size]).cmd_tuples
194-
printf "Deleted #{count} actor#{count == 1 ? "" : "s"} (total = #{total_deleted + count})"
263+
printf "Deleted #{count} actor#{count == 1 ? "" : "s"} (total = #{total_actors_deleted + count})"
264+
count
265+
end
266+
267+
total_actors_deleted += deletion_count
268+
break if deletion_count == 0
269+
end
270+
puts "Total auth_actors removed: #{total_actors_deleted}"
271+
272+
total_object_deleted = 0
273+
loop do
274+
deletion_count = timed "Processing batch of #{batch_size} unknown auth_objects. " do
275+
count = db.exec(CLEANUP_SQL2, [batch_size]).cmd_tuples
276+
printf "Deleted #{count} object#{count == 1 ? "" : "s"} (total = #{total_object_deleted + count})"
195277
count
196278
end
197279

198-
total_deleted += deletion_count
280+
total_object_deleted += deletion_count
199281
break if deletion_count == 0
200282
end
201-
puts "Total auth_actors removed: #{total_deleted}"
283+
puts "Total auth_objects removed: #{total_object_deleted}"
202284
end
203285

204286
def timed(description)

0 commit comments

Comments
 (0)