Skip to content

Commit 242e81e

Browse files
committed
HBASE-30231 LoadIncrementalHFiles failed for MetricsException: Metrics source RegionServer,Sub=IO already exists (#8834)
MetricsIO.getInstance() lazily registers the "RegionServer,sub=IO" metrics source on the first HFile read or write. Hadoop's MetricsSystemImpl.register() reserves the source name before registration has fully completed, so if anything fails after that point the singleton is never assigned and every later call fails with "Metrics source RegionServer,sub=IO already exists!". This breaks all HFile reads and writes for the rest of the JVM's life, as seen in a bulk load client splitting HFiles in LoadIncrementalHFiles. - If creating the source fails, log a warning once and use a MetricsIO backed by a no-op MetricsIOSource, so a metrics problem can never fail HFile I/O. - Make IO metrics opt-in: MetricsIO.getInstance() returns a shared no-op instance until MetricsIO.enable() is called, which HBaseServerBase does in its constructor, covering both HRegionServer and HMaster (the master reads and writes HFiles through its local MasterRegion). Clients and tools that read or write HFiles (bulk load, MapReduce and Spark tasks, HFilePrettyPrinter, ...) no longer start the Hadoop metrics system or register a RegionServer source. The real source is still created lazily on first use (HBASE-27966), so server behavior is unchanged. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Wellington Chevreuil <[email protected]> Signed-off-by: Xiao Liu <[email protected]> Signed-off-by: David Paksy <[email protected]>
1 parent b4e0358 commit 242e81e

4 files changed

Lines changed: 249 additions & 1 deletion

File tree

‎hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.hadoop.hbase.executor.ExecutorService;
5252
import org.apache.hadoop.hbase.fs.HFileSystem;
5353
import org.apache.hadoop.hbase.http.InfoServer;
54+
import org.apache.hadoop.hbase.io.MetricsIO;
5455
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
5556
import org.apache.hadoop.hbase.ipc.RpcServerInterface;
5657
import org.apache.hadoop.hbase.keymeta.KeyManagementService;
@@ -263,6 +264,10 @@ public HBaseServerBase(Configuration conf, String name) throws IOException {
263264
final Span span = TraceUtil.createSpan("HBaseServerBase.cxtor");
264265
try (Scope ignored = span.makeCurrent()) {
265266
this.conf = conf;
267+
// Report HFile IO metrics only from server processes, not from clients and tools that also
268+
// read or write HFiles. Done here so it covers both HRegionServer and HMaster, which reads
269+
// and writes HFiles through its local MasterRegion. See HBASE-30231.
270+
MetricsIO.enable();
266271
this.eventLoopGroupConfig =
267272
NettyEventLoopGroupConfig.setup(conf, getClass().getSimpleName() + "-EventLoopGroup");
268273
this.startcode = EnvironmentEdgeManager.currentTime();

‎hbase-server/src/main/java/org/apache/hadoop/hbase/io/MetricsIO.java‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,26 @@
1818
package org.apache.hadoop.hbase.io;
1919

2020
import com.google.errorprone.annotations.RestrictedApi;
21+
import java.util.function.Supplier;
2122
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
2223
import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory;
2324
import org.apache.yetus.audience.InterfaceAudience;
25+
import org.slf4j.Logger;
26+
import org.slf4j.LoggerFactory;
2427

2528
@InterfaceAudience.Private
2629
public class MetricsIO {
2730

31+
private static final Logger LOG = LoggerFactory.getLogger(MetricsIO.class);
32+
33+
/**
34+
* Shared instance returned while IO metrics are disabled. It discards all updates and never
35+
* registers anything with the metrics system.
36+
*/
37+
private static final MetricsIO DISABLED_INSTANCE =
38+
new MetricsIO(new NoOpMetricsIOSource(), new MetricsIOWrapperImpl());
39+
40+
private static volatile boolean enabled = false;
2841
private static volatile MetricsIO instance;
2942
private final MetricsIOSource source;
3043
private final MetricsIOWrapper wrapper;
@@ -41,21 +54,73 @@ public class MetricsIO {
4154
this.wrapper = wrapper;
4255
}
4356

57+
/**
58+
* Enables IO metrics for this process. Until this is called, {@link #getInstance()} returns an
59+
* instance that discards all updates, so processes that only read or write HFiles as a client
60+
* (bulk load, MapReduce tasks, command line tools) do not start the Hadoop metrics system or
61+
* register a "RegionServer" metrics source. Server processes call this during startup. The
62+
* metrics source itself is still created lazily, on first use after this call. See HBASE-30231
63+
* for more details.
64+
*/
65+
public static void enable() {
66+
enabled = true;
67+
}
68+
69+
/** Returns whether IO metrics have been enabled for this process. */
70+
public static boolean isEnabled() {
71+
return enabled;
72+
}
73+
74+
@RestrictedApi(explanation = "Should only be called in tests", link = "",
75+
allowedOnPath = ".*/src/test/.*")
76+
public static synchronized void resetForTesting() {
77+
enabled = false;
78+
instance = null;
79+
}
80+
4481
/**
4582
* Get a static instance for the MetricsIO so that accessors access the same instance. We want to
4683
* lazy initialize so that correct process name is in place. See HBASE-27966 for more details.
84+
* <p>
85+
* If IO metrics have not been enabled with {@link #enable()}, a shared instance that discards all
86+
* updates is returned. If the metrics source cannot be created, a no-op source is used instead,
87+
* so that a metrics failure never fails HFile reads or writes. See HBASE-30231 for more details.
4788
*/
4889
public static MetricsIO getInstance() {
90+
if (!enabled) {
91+
return DISABLED_INSTANCE;
92+
}
4993
if (instance == null) {
5094
synchronized (MetricsIO.class) {
5195
if (instance == null) {
52-
instance = new MetricsIO(new MetricsIOWrapperImpl());
96+
MetricsIOWrapper wrapper = new MetricsIOWrapperImpl();
97+
instance = createInstance(() -> new MetricsIO(wrapper), wrapper);
5398
}
5499
}
55100
}
56101
return instance;
57102
}
58103

104+
/**
105+
* Creates a {@link MetricsIO} with the given supplier, falling back to one backed by a no-op
106+
* {@link MetricsIOSource} if the supplier fails.
107+
* <p>
108+
* Registering the metrics source with the Hadoop metrics system reserves its name before the
109+
* registration has fully completed. If anything fails after that point, every later attempt to
110+
* register the same source fails with "Metrics source ... already exists!". Retrying would
111+
* therefore fail on every call for the rest of the JVM's life, so we fall back to a no-op source
112+
* once and never retry.
113+
*/
114+
static MetricsIO createInstance(Supplier<MetricsIO> supplier, MetricsIOWrapper wrapper) {
115+
try {
116+
return supplier.get();
117+
} catch (RuntimeException | LinkageError e) {
118+
LOG.warn("Failed to initialize the {} metrics source; FileSystem IO metrics will not be"
119+
+ " reported by this process", MetricsIOSource.METRICS_JMX_CONTEXT, e);
120+
return new MetricsIO(new NoOpMetricsIOSource(), wrapper);
121+
}
122+
}
123+
59124
public MetricsIOSource getMetricsSource() {
60125
return source;
61126
}
@@ -79,4 +144,75 @@ public void updateFsWriteTime(long t) {
79144
public void incrSlowFsRead() {
80145
source.incrSlowFsRead();
81146
}
147+
148+
/**
149+
* A {@link MetricsIOSource} that discards all updates. Used while IO metrics are disabled, or
150+
* when the real metrics source could not be created.
151+
*/
152+
static final class NoOpMetricsIOSource implements MetricsIOSource {
153+
154+
@Override
155+
public void updateFsReadTime(long t) {
156+
}
157+
158+
@Override
159+
public void updateFsPReadTime(long t) {
160+
}
161+
162+
@Override
163+
public void updateFsWriteTime(long t) {
164+
}
165+
166+
@Override
167+
public void incrSlowFsRead() {
168+
}
169+
170+
@Override
171+
public void init() {
172+
}
173+
174+
@Override
175+
public void setGauge(String gaugeName, long value) {
176+
}
177+
178+
@Override
179+
public void incGauge(String gaugeName, long delta) {
180+
}
181+
182+
@Override
183+
public void decGauge(String gaugeName, long delta) {
184+
}
185+
186+
@Override
187+
public void removeMetric(String key) {
188+
}
189+
190+
@Override
191+
public void incCounters(String counterName, long delta) {
192+
}
193+
194+
@Override
195+
public void updateHistogram(String name, long value) {
196+
}
197+
198+
@Override
199+
public String getMetricsContext() {
200+
return METRICS_CONTEXT;
201+
}
202+
203+
@Override
204+
public String getMetricsDescription() {
205+
return METRICS_DESCRIPTION;
206+
}
207+
208+
@Override
209+
public String getMetricsJmxContext() {
210+
return METRICS_JMX_CONTEXT;
211+
}
212+
213+
@Override
214+
public String getMetricsName() {
215+
return METRICS_NAME;
216+
}
217+
}
82218
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestMetricsIO.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,20 @@
1717
*/
1818
package org.apache.hadoop.hbase.io;
1919

20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertFalse;
22+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
23+
import static org.junit.jupiter.api.Assertions.assertNotSame;
24+
import static org.junit.jupiter.api.Assertions.assertSame;
25+
import static org.junit.jupiter.api.Assertions.assertTrue;
26+
2027
import org.apache.hadoop.hbase.CompatibilityFactory;
2128
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
2229
import org.apache.hadoop.hbase.testclassification.MiscTests;
2330
import org.apache.hadoop.hbase.testclassification.SmallTests;
31+
import org.apache.hadoop.metrics2.MetricsException;
32+
import org.junit.jupiter.api.AfterEach;
33+
import org.junit.jupiter.api.BeforeEach;
2434
import org.junit.jupiter.api.Tag;
2535
import org.junit.jupiter.api.Test;
2636

@@ -30,6 +40,45 @@ public class TestMetricsIO {
3040

3141
public MetricsAssertHelper HELPER = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
3242

43+
@BeforeEach
44+
public void setUp() {
45+
// Several tests register the same source, which the metrics system only allows in this mode.
46+
HELPER.init();
47+
MetricsIO.resetForTesting();
48+
}
49+
50+
@AfterEach
51+
public void tearDown() {
52+
MetricsIO.resetForTesting();
53+
}
54+
55+
@Test
56+
public void testDisabledByDefault() {
57+
assertFalse(MetricsIO.isEnabled());
58+
MetricsIO metrics = MetricsIO.getInstance();
59+
assertInstanceOf(MetricsIO.NoOpMetricsIOSource.class, metrics.getMetricsSource());
60+
assertSame(metrics, MetricsIO.getInstance());
61+
// Updates must be silently discarded while disabled.
62+
metrics.updateFsReadTime(100);
63+
metrics.updateFsPreadTime(200);
64+
metrics.updateFsWriteTime(300);
65+
metrics.incrSlowFsRead();
66+
}
67+
68+
@Test
69+
public void testEnable() {
70+
MetricsIO disabled = MetricsIO.getInstance();
71+
MetricsIO.enable();
72+
assertTrue(MetricsIO.isEnabled());
73+
MetricsIO enabled = MetricsIO.getInstance();
74+
assertNotSame(disabled, enabled);
75+
assertFalse(enabled.getMetricsSource() instanceof MetricsIO.NoOpMetricsIOSource);
76+
assertSame(enabled, MetricsIO.getInstance());
77+
78+
enabled.updateFsWriteTime(100);
79+
HELPER.assertCounter("fsWriteTime_numOps", 1, enabled.getMetricsSource());
80+
}
81+
3382
@Test
3483
public void testMetrics() {
3584
MetricsIO metrics = new MetricsIO(new MetricsIOWrapper() {
@@ -54,4 +103,42 @@ public long getChecksumFailures() {
54103
HELPER.assertCounter("fsPReadTime_numOps", 1, metrics.getMetricsSource());
55104
HELPER.assertCounter("fsWriteTime_numOps", 3, metrics.getMetricsSource());
56105
}
106+
107+
@Test
108+
public void testCreateInstanceUsesSupplierResult() {
109+
MetricsIOWrapper wrapper = () -> 0;
110+
MetricsIO expected = new MetricsIO(new MetricsIO.NoOpMetricsIOSource(), wrapper);
111+
assertSame(expected, MetricsIO.createInstance(() -> expected, wrapper));
112+
}
113+
114+
@Test
115+
public void testFallbackToNoOpSourceOnRuntimeException() {
116+
MetricsIOWrapper wrapper = () -> 0;
117+
MetricsIO metrics = MetricsIO.createInstance(() -> {
118+
// What a second registration attempt of the source fails with, see HBASE-30231
119+
throw new MetricsException("Metrics source RegionServer,sub=IO already exists!");
120+
}, wrapper);
121+
assertFallback(metrics, wrapper);
122+
}
123+
124+
@Test
125+
public void testFallbackToNoOpSourceOnLinkageError() {
126+
MetricsIOWrapper wrapper = () -> 0;
127+
MetricsIO metrics = MetricsIO.createInstance(() -> {
128+
throw new ExceptionInInitializerError("simulated static initializer failure");
129+
}, wrapper);
130+
assertFallback(metrics, wrapper);
131+
}
132+
133+
private static void assertFallback(MetricsIO metrics, MetricsIOWrapper wrapper) {
134+
assertInstanceOf(MetricsIO.NoOpMetricsIOSource.class, metrics.getMetricsSource());
135+
assertSame(wrapper, metrics.getWrapper());
136+
assertEquals(MetricsIOSource.METRICS_JMX_CONTEXT,
137+
metrics.getMetricsSource().getMetricsJmxContext());
138+
// Updates must be silently discarded rather than failing the caller's HFile read or write.
139+
metrics.updateFsReadTime(100);
140+
metrics.updateFsPreadTime(200);
141+
metrics.updateFsWriteTime(300);
142+
metrics.incrSlowFsRead();
143+
}
57144
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
2121
import static org.junit.jupiter.api.Assertions.assertFalse;
2222
import static org.junit.jupiter.api.Assertions.assertNull;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
2324

2425
import java.io.IOException;
2526
import org.apache.hadoop.conf.Configuration;
@@ -28,6 +29,7 @@
2829
import org.apache.hadoop.hbase.HConstants;
2930
import org.apache.hadoop.hbase.Waiter;
3031
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
32+
import org.apache.hadoop.hbase.io.MetricsIO;
3133
import org.apache.hadoop.hbase.testclassification.MasterTests;
3234
import org.apache.hadoop.hbase.testclassification.MediumTests;
3335
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -117,6 +119,24 @@ public void testStopDuringStart() throws IOException, KeeperException, Interrupt
117119
master.join();
118120
}
119121

122+
/**
123+
* The master reads and writes HFiles through its local MasterRegion, so it must enable IO metrics
124+
* even when no RegionServer runs in the same JVM. See HBASE-30231.
125+
*/
126+
@Test
127+
public void testMasterEnablesMetricsIO() throws IOException, InterruptedException {
128+
MetricsIO.resetForTesting();
129+
HMaster master = new HMaster(TESTUTIL.getConfiguration());
130+
master.start();
131+
try {
132+
assertTrue(MetricsIO.isEnabled());
133+
} finally {
134+
master.stopMaster();
135+
master.join();
136+
MetricsIO.resetForTesting();
137+
}
138+
}
139+
120140
@Test
121141
public void testMasterInitWithSameClientServerZKQuorum() throws Exception {
122142
Configuration conf = new Configuration(TESTUTIL.getConfiguration());

0 commit comments

Comments
 (0)