Skip to content

Commit 400194c

Browse files
morningmanclaude
andcommitted
[fix](ranger) Refuse after a load that fails past publication, and stop waiting for the load once the controller is closed
Two review findings on 6669fb6, the plugin that loads on its own thread. RangerBasePlugin.init() installs the policy engine and starts its refresher before it initializes the chained plugins, so a chained plugin that fails to start throws out of the load with a live engine behind it - and the catch in load() released the latch over it, logging that every check would be refused while that engine went on answering with part of the configured authorization missing. The catch now records the failure and stops what the load had installed before the latch is released, and every answer refuses (null, which each caller already reads as a refusal) while the plugin is in that state. A controller closed while its check waited for the load - a catalog detached during a load an unreachable admin holds up - sat out the plugin's REST timeouts before the lifecycle fence refused it. awaitLoaded(BooleanSupplier) gives up as soon as the caller says so, and the controller's three entry points wait "unless closed": refused at once through the existing fence, read lock and all, while the load goes on for whoever else holds the plugin. Also: when the load ends with policies but no user store while groups are on, a WARN names the consequence - requests carry no groups until one arrives - rather than leaving it to the "user store version -1" in the load line. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_0191LreyBauJV7K7sZAbrB27
1 parent 6669fb6 commit 400194c

4 files changed

Lines changed: 253 additions & 20 deletions

File tree

‎fe/fe-authorization/fe-authorization-plugins/fe-authorization-plugin-ranger-common/src/main/java/org/apache/doris/catalog/authorizer/ranger/BackgroundLoadedRangerPlugin.java‎

Lines changed: 68 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.concurrent.TimeUnit;
3232
import java.util.concurrent.atomic.AtomicBoolean;
3333
import java.util.concurrent.atomic.AtomicReference;
34+
import java.util.function.BooleanSupplier;
3435

3536
/**
3637
* The plugin a Doris Ranger source answers out of: a {@link RangerBasePlugin} that loads without holding up
@@ -53,10 +54,16 @@
5354
* <p>What a check waits for is exactly what the constructor used to guarantee: that the load has
5455
* <em>ended</em> - with the policies from the admin, or from the local cache when the admin could not be
5556
* reached, or with nothing at all, in which case the engine is null and {@code RangerAccessController}
56-
* refuses, as it always has. It is not shortened by a timeout of its own: the load is bounded by the REST
57-
* timeouts the operator already tunes, and answering out of an empty engine before it has ended would be
58-
* refusing checks the policies are about to allow - and, worse, passing ones a policy written against a
59-
* group is about to deny.
57+
* refuses, as it always has. A load that <em>threw</em> ends the same way, whatever it had installed by
58+
* then: what {@code RangerBasePlugin.init()} does about an admin it cannot reach is logged and survived
59+
* inside it, so a throw is something else - a broken audit configuration, a chained plugin that could not
60+
* start - and where the constructor used to fail the FE's start with it, this plugin stops what the load
61+
* did publish and refuses every check until the FE is restarted, see {@link #load()}. The wait is not
62+
* shortened by a timeout of its own: the load is bounded by the REST timeouts the operator already tunes,
63+
* and answering out of an empty engine before it has ended would be refusing checks the policies are about
64+
* to allow - and, worse, passing ones a policy written against a group is about to deny. What does cut it
65+
* short is the caller having no use for the answer any more: a controller closed while its check waits
66+
* refuses, and {@link #awaitLoaded(BooleanSupplier)} is how it stops waiting.
6067
*
6168
* <p>Stopping it is {@link #cleanup()}, as before. Stopped while still loading - a {@code CREATE CATALOG}
6269
* dry run, the loser of a race in a factory - it finishes the load first and stops itself then, on the
@@ -72,6 +79,8 @@
7279
*/
7380
public abstract class BackgroundLoadedRangerPlugin extends RangerBasePlugin {
7481
private static final Logger LOG = LogManager.getLogger(BackgroundLoadedRangerPlugin.class);
82+
/** How often {@link #awaitLoaded(BooleanSupplier)} asks whether to give up. */
83+
private static final long GIVE_UP_CHECK_MS = 100;
7584

7685
/** Released once the first load has ended, however it ended. */
7786
private final CountDownLatch loaded = new CountDownLatch(1);
@@ -81,6 +90,8 @@ public abstract class BackgroundLoadedRangerPlugin extends RangerBasePlugin {
8190
private volatile boolean stopRequested;
8291
/** Whether the stop has run. It runs once, from whichever of {@link #cleanup()} and the loader is last. */
8392
private final AtomicBoolean stopped = new AtomicBoolean();
93+
/** Whether the first load threw. Every answer is refused then; see {@link #load()}. */
94+
private volatile boolean failed;
8495

8596
protected BackgroundLoadedRangerPlugin(String serviceType, String serviceName, String appId) {
8697
super(serviceType, serviceName, appId);
@@ -119,13 +130,30 @@ private void load() {
119130
+ " version {}", getServiceName(),
120131
TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos), getPoliciesVersion(),
121132
getRolesVersion(), getUserStoreVersion());
133+
if (getPolicyEngine() != null && RangerUserStoreGroups.enabledFor(getConfig())
134+
&& getUserStoreVersion() < 0) {
135+
// Policies to answer out of, but no user store to read groups from: the admin could not be
136+
// reached for it and none was cached. Requests carry no groups until one arrives - the
137+
// enricher keeps asking - which is what this source sent before groups were attached at all,
138+
// and which a policy item written against a group, a deny included, does not match.
139+
LOG.warn("Ranger service {} loaded its policies but no user store: until one arrives,"
140+
+ " requests carry no groups and policy items written against a group do not apply",
141+
getServiceName());
142+
}
122143
} catch (Throwable e) {
123144
// Everything RangerBasePlugin.init() does about an admin it cannot reach is logged and survived
124-
// inside it, so this is something else - a broken audit configuration, say. It used to fail the
125-
// FE's start; now it fails every check against this service until the FE is restarted, which
126-
// RangerAccessController reports on each one as an engine that is not initialized.
145+
// inside it, so this is something else - a broken audit configuration, a chained plugin that
146+
// could not start. It used to fail the FE's start; now it fails every check against this
147+
// service until the FE is restarted, which RangerAccessController reports on each one as an
148+
// engine that is not initialized. Refusing takes both of the following: the flag, read by every
149+
// answer, and stopping what the load had installed before it threw - RangerBasePlugin.init()
150+
// publishes the policy engine, refresher and all, before it initializes the chained plugins, and
151+
// left running that engine would answer checks with part of the configured authorization
152+
// missing, whatever the line below says.
153+
failed = true;
127154
LOG.error("Ranger service {} failed to load; every check against it is refused until the FE is"
128155
+ " restarted", getServiceName(), e);
156+
stopOnce();
129157
} finally {
130158
loaded.countDown();
131159
if (stopRequested) {
@@ -160,6 +188,26 @@ public void awaitLoaded() {
160188
}
161189
}
162190

191+
/**
192+
* As {@link #awaitLoaded()}, for a caller that may stop needing the answer while it waits: returns once
193+
* the load has ended or {@code giveUp} says so, whichever is first, the latter asked every
194+
* {@value #GIVE_UP_CHECK_MS} ms. A controller closed while its check waits is such a caller - it
195+
* refuses, and the load it was waiting for goes on for whoever else holds the plugin.
196+
*/
197+
public void awaitLoaded(BooleanSupplier giveUp) {
198+
if (loader.get() == null || isLoaded()) {
199+
return;
200+
}
201+
try {
202+
boolean ended = false;
203+
while (!ended && !giveUp.getAsBoolean()) {
204+
ended = loaded.await(GIVE_UP_CHECK_MS, TimeUnit.MILLISECONDS);
205+
}
206+
} catch (InterruptedException e) {
207+
Thread.currentThread().interrupt();
208+
}
209+
}
210+
163211
/**
164212
* Whether the first load has ended - not whether it found anything, see {@link #awaitLoaded()} - and so
165213
* false for a plugin that was never initialized, which has no load to end.
@@ -168,6 +216,12 @@ public boolean isLoaded() {
168216
return loaded.getCount() == 0;
169217
}
170218

219+
/** Whether the first load threw, after which every answer is a refusal; see {@link #load()}. */
220+
@VisibleForTesting
221+
boolean isFailed() {
222+
return failed;
223+
}
224+
171225
/**
172226
* Stops the plugin's threads, or - while the first load is still running - arranges for the loader to
173227
* stop them once it has ended, and returns at once.
@@ -212,34 +266,35 @@ public void setPolicies(ServicePolicies policies) {
212266
super.setPolicies(policies);
213267
}
214268

215-
// Every answer waits for the first load to end. These four are the ones Doris asks; the one-argument
216-
// isAccessAllowed overloads arrive here through RangerBasePlugin.
269+
// Every answer waits for the first load to end, and a load that failed is answered with null - what
270+
// RangerBasePlugin answers with no engine, and what every caller reads as a refusal. These four are the
271+
// ones Doris asks; the one-argument isAccessAllowed overloads arrive here through RangerBasePlugin.
217272

218273
@Override
219274
public RangerAccessResult isAccessAllowed(RangerAccessRequest request,
220275
RangerAccessResultProcessor resultProcessor) {
221276
awaitLoaded();
222-
return super.isAccessAllowed(request, resultProcessor);
277+
return failed ? null : super.isAccessAllowed(request, resultProcessor);
223278
}
224279

225280
@Override
226281
public Collection<RangerAccessResult> isAccessAllowed(Collection<RangerAccessRequest> requests,
227282
RangerAccessResultProcessor resultProcessor) {
228283
awaitLoaded();
229-
return super.isAccessAllowed(requests, resultProcessor);
284+
return failed ? null : super.isAccessAllowed(requests, resultProcessor);
230285
}
231286

232287
@Override
233288
public RangerAccessResult evalDataMaskPolicies(RangerAccessRequest request,
234289
RangerAccessResultProcessor resultProcessor) {
235290
awaitLoaded();
236-
return super.evalDataMaskPolicies(request, resultProcessor);
291+
return failed ? null : super.evalDataMaskPolicies(request, resultProcessor);
237292
}
238293

239294
@Override
240295
public RangerAccessResult evalRowFilterPolicies(RangerAccessRequest request,
241296
RangerAccessResultProcessor resultProcessor) {
242297
awaitLoaded();
243-
return super.evalRowFilterPolicies(request, resultProcessor);
298+
return failed ? null : super.evalRowFilterPolicies(request, resultProcessor);
244299
}
245300
}

‎fe/fe-authorization/fe-authorization-plugins/fe-authorization-plugin-ranger-common/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,11 @@
5959
* read the table whole and in the clear.
6060
*
6161
* <p>A plugin still on its first load is neither: it is about to have an answer, and every question here
62-
* waits for it ({@link BackgroundLoadedRangerPlugin#awaitLoaded()}) before anything else - before the
63-
* lifecycle fence above all, so that a load the Ranger admin is slow to serve is waited for with no lock
64-
* held, and closing this controller meanwhile is not queued behind it.
62+
* waits for it ({@link BackgroundLoadedRangerPlugin#awaitLoaded(java.util.function.BooleanSupplier)})
63+
* before anything else - before the lifecycle fence above all, so that a load the Ranger admin is slow to
64+
* serve is waited for with no lock held, and closing this controller meanwhile is not queued behind it. Nor
65+
* is the wait queued behind the close: a controller closed while its check waits stops waiting and refuses,
66+
* as it would have after the load.
6567
*/
6668
public abstract class RangerAccessController implements AuthorizationPlugin {
6769
private static final Logger LOG = LogManager.getLogger(RangerAccessController.class);
@@ -185,10 +187,22 @@ protected interface PluginCheck {
185187
@Override
186188
public final void checkPrivilege(AuthorizedSubject subject, AuthorizedResource resource,
187189
AccessRequirement requirement, AccessContext context) throws AccessDeniedException {
188-
getPlugin().awaitLoaded();
190+
awaitPluginUnlessClosed();
189191
checkWhileOpen(resource, () -> checkPrivilegeInternal(subject, resource, requirement, context));
190192
}
191193

194+
/**
195+
* Waits for the Ranger plugin's first load to end, unless this controller is closed first.
196+
*
197+
* <p>Before the lifecycle fence and with no lock held, see the class comment. Closed, this controller
198+
* refuses whatever the load brings, so there is nothing to wait for: a query holding a controller its
199+
* catalog has since let go of is refused at once, not after the REST timeouts of a Ranger admin that is
200+
* not answering - the plugin's own {@code cleanup()} deliberately does not wait for that load either.
201+
*/
202+
private void awaitPluginUnlessClosed() {
203+
getPlugin().awaitLoaded(() -> closed);
204+
}
205+
192206
/** What this Ranger service decides about {@code resource}, asked only while this source is open. */
193207
protected abstract void checkPrivilegeInternal(AuthorizedSubject subject, AuthorizedResource resource,
194208
AccessRequirement requirement, AccessContext context) throws AccessDeniedException;
@@ -339,7 +353,7 @@ private static String policyIdent(RangerAccessResult policy) {
339353
@Override
340354
public List<RowFilterSpec> getRowFilters(AuthorizedSubject subject, AuthorizedResource.Table table,
341355
AccessContext context) {
342-
getPlugin().awaitLoaded();
356+
awaitPluginUnlessClosed();
343357
return whileOpen(() -> evalRowFilterPolicies(subject, table, context));
344358
}
345359

@@ -389,7 +403,7 @@ private List<RowFilterSpec> evalRowFilterPolicies(AuthorizedSubject subject,
389403
@Override
390404
public Map<String, DataMaskSpec> getDataMasks(AuthorizedSubject subject, AuthorizedResource.Table table,
391405
Set<String> columns, AccessContext context) {
392-
getPlugin().awaitLoaded();
406+
awaitPluginUnlessClosed();
393407
return whileOpen(() -> {
394408
Map<String, DataMaskSpec> masks = new HashMap<>();
395409
for (String column : columns) {

‎fe/fe-authorization/fe-authorization-plugins/fe-authorization-plugin-ranger-doris/src/test/java/org/apache/doris/catalog/authorizer/ranger/BackgroundLoadedRangerPluginTest.java‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@
1919

2020
import com.google.common.collect.ImmutableMap;
2121
import com.google.common.collect.ImmutableSet;
22+
import org.apache.ranger.plugin.model.RangerServiceDef;
2223
import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
2324
import org.apache.ranger.plugin.service.RangerAuthContext;
2425
import org.apache.ranger.plugin.util.RangerUserStore;
26+
import org.apache.ranger.plugin.util.ServicePolicies;
2527
import org.junit.jupiter.api.AfterEach;
2628
import org.junit.jupiter.api.Assertions;
2729
import org.junit.jupiter.api.Test;
@@ -51,13 +53,29 @@ private static final class Loading extends BackgroundLoadedRangerPlugin {
5153
private final boolean clearsStateFirst;
5254
/** The user store the load installs, standing for the one the enricher downloads; null for none. */
5355
private final RangerUserStore userStore;
56+
/**
57+
* Whether the load installs a policy engine and then throws - what {@code RangerBasePlugin.init()}
58+
* does when a chained plugin fails to start, after the root engine is up and being refreshed.
59+
*/
60+
private final boolean publishesThenThrows;
61+
/** Whether an engine was there when the load threw; what the failure is supposed to take down. */
62+
private final AtomicBoolean enginePublished = new AtomicBoolean();
5463

5564
private Loading(boolean clearsStateFirst, RangerUserStore userStore) {
65+
this(clearsStateFirst, userStore, false);
66+
}
67+
68+
private Loading(boolean clearsStateFirst, RangerUserStore userStore, boolean publishesThenThrows) {
5669
// Service type "test" reads ranger-test-*.xml, none of which exist here, and no service name:
5770
// nothing about a Ranger admin is configured, which is fine for a load this class performs itself.
5871
super("test", null, null);
5972
this.clearsStateFirst = clearsStateFirst;
6073
this.userStore = userStore;
74+
this.publishesThenThrows = publishesThenThrows;
75+
if (publishesThenThrows) {
76+
// The engine built below would otherwise ask a Ranger admin for the user store; there is none.
77+
getConfig().set("ranger.plugin.test.use.rangerGroups", "false");
78+
}
6179
}
6280

6381
@Override
@@ -75,6 +93,22 @@ protected void firstLoad() {
7593
if (userStore != null) {
7694
getPluginContext().setAuthContext(new RangerAuthContext(null, null, null, userStore));
7795
}
96+
if (publishesThenThrows) {
97+
setPolicies(emptyPolicies());
98+
enginePublished.set(getPolicyEngine() != null);
99+
throw new IllegalStateException("a chained plugin failed to start");
100+
}
101+
}
102+
103+
/** Policies with nothing in them, enough for RangerBasePlugin to build and install an engine. */
104+
private static ServicePolicies emptyPolicies() {
105+
RangerServiceDef serviceDef = new RangerServiceDef();
106+
serviceDef.setName("test");
107+
ServicePolicies policies = new ServicePolicies();
108+
policies.setServiceName("test");
109+
policies.setServiceDef(serviceDef);
110+
policies.setPolicyVersion(1L);
111+
return policies;
78112
}
79113

80114
private void letTheLoadEnd() {
@@ -89,7 +123,11 @@ private Loading loading() {
89123
}
90124

91125
private Loading loading(boolean clearsStateFirst, RangerUserStore userStore) {
92-
plugin = new Loading(clearsStateFirst, userStore);
126+
return loading(clearsStateFirst, userStore, false);
127+
}
128+
129+
private Loading loading(boolean clearsStateFirst, RangerUserStore userStore, boolean publishesThenThrows) {
130+
plugin = new Loading(clearsStateFirst, userStore, publishesThenThrows);
93131
return plugin;
94132
}
95133

@@ -163,6 +201,58 @@ public void testGroupsAreReadAfterTheLoad() throws Exception {
163201
Assertions.assertEquals(ImmutableSet.of("analysts", "etl"), within(groups, 10));
164202
}
165203

204+
/**
205+
* A load that throws after it has installed an engine - {@code RangerBasePlugin.init()} publishes the
206+
* policy engine and starts its refresher before it initializes the chained plugins - is a failed load
207+
* all the same: what it installed is stopped, and every answer is a refusal, as the log line says.
208+
*/
209+
@Test
210+
public void testALoadThatFailsAfterPublishingRefusesEverything() throws Exception {
211+
Loading plugin = loading(false, null, true);
212+
plugin.init();
213+
Assertions.assertTrue(plugin.loadStarted.await(10, TimeUnit.SECONDS));
214+
215+
CompletableFuture<Object> answer = CompletableFuture.supplyAsync(
216+
() -> plugin.isAccessAllowed(new RangerAccessRequestImpl()));
217+
assertStillWaiting(answer);
218+
plugin.letTheLoadEnd();
219+
220+
Assertions.assertNull(within(answer, 10), "answered out of the engine the failed load left behind");
221+
Assertions.assertTrue(plugin.enginePublished.get(), "the load never published an engine to take down");
222+
Assertions.assertTrue(plugin.isLoaded());
223+
Assertions.assertTrue(plugin.isFailed());
224+
Assertions.assertTrue(plugin.isStopped(), "the engine and refresher of a failed load were left running");
225+
Assertions.assertEquals(-1L, plugin.getPoliciesVersion(), "the engine the failed load left is still there");
226+
Assertions.assertNull(plugin.isAccessAllowed(new RangerAccessRequestImpl()));
227+
Assertions.assertNull(plugin.evalRowFilterPolicies(new RangerAccessRequestImpl(), null));
228+
Assertions.assertNull(plugin.evalDataMaskPolicies(new RangerAccessRequestImpl(), null));
229+
// Stopping it again, as the factory will, is a no-op rather than a second stop.
230+
plugin.cleanup();
231+
Assertions.assertTrue(plugin.isStopped());
232+
}
233+
234+
/**
235+
* A caller that stops needing the answer - a controller closed while its check waits - stops waiting,
236+
* while the load goes on for whoever else holds the plugin.
237+
*/
238+
@Test
239+
public void testAWaitGivesUpWhenAsked() throws Exception {
240+
Loading plugin = loading();
241+
plugin.init();
242+
Assertions.assertTrue(plugin.loadStarted.await(10, TimeUnit.SECONDS));
243+
AtomicBoolean giveUp = new AtomicBoolean();
244+
245+
CompletableFuture<Void> waiting = CompletableFuture.runAsync(() -> plugin.awaitLoaded(giveUp::get));
246+
247+
assertStillWaiting(waiting);
248+
giveUp.set(true);
249+
within(waiting, 10);
250+
Assertions.assertFalse(plugin.isLoaded(), "the load was cut short with the waiter");
251+
plugin.letTheLoadEnd();
252+
plugin.awaitLoaded(() -> false);
253+
Assertions.assertTrue(plugin.isLoaded());
254+
}
255+
166256
/** A plugin never initialized - a test's own, answering out of its overrides - has nothing to wait for. */
167257
@Test
168258
public void testNeverInitializedHasNothingToWaitFor() {

0 commit comments

Comments
 (0)