Skip to content

Commit 9ea48c1

Browse files
authored
HBASE-30231 LoadIncrementalHFiles failed for MetricsException: Metrics source RegionServer,Sub=IO already exists (#8834)
* HBASE-30231 Apply fix #1 and #2 * HBASE-30231 Apply changes for the master branch * HBASE-30231. Spotless apply
1 parent ed79a76 commit 9ea48c1

4 files changed

Lines changed: 249 additions & 1 deletion

File tree

‎hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.hadoop.hbase.executor.ExecutorService;
5252
import org.apache.hadoop.hbase.fs.HFileSystem;
5353
import org.apache.hadoop.hbase.http.InfoServer;
54+
import org.apache.hadoop.hbase.io.MetricsIO;
5455
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
5556
import org.apache.hadoop.hbase.ipc.RpcServerInterface;
5657
import org.apache.hadoop.hbase.keymeta.KeyManagementService;
@@ -263,6 +264,10 @@ public HBaseServerBase(Configuration conf, String name) throws IOException {
263264
final Span span = TraceUtil.createSpan("HBaseServerBase.cxtor");
264265
try (Scope ignored = span.makeCurrent()) {
265266
this.conf = conf;
267+
// Report HFile IO metrics only from server processes, not from clients and tools that also
268+
// read or write HFiles. Done here so it covers both HRegionServer and HMaster, which reads
269+
// and writes HFiles through its local MasterRegion. See HBASE-30231.
270+
MetricsIO.enable();
266271
this.eventLoopGroupConfig =
267272
NettyEventLoopGroupConfig.setup(conf, getClass().getSimpleName() + "-EventLoopGroup");
268273
this.startcode = EnvironmentEdgeManager.currentTime();

‎hbase-server/src/main/java/org/apache/hadoop/hbase/io/MetricsIO.java‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,26 @@
1818
package org.apache.hadoop.hbase.io;
1919

2020
import com.google.errorprone.annotations.RestrictedApi;
21+
import java.util.function.Supplier;
2122
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
2223
import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory;
2324
import org.apache.yetus.audience.InterfaceAudience;
25+
import org.slf4j.Logger;
26+
import org.slf4j.LoggerFactory;
2427

2528
@InterfaceAudience.Private
2629
public class MetricsIO {
2730

31+
private static final Logger LOG = LoggerFactory.getLogger(MetricsIO.class);
32+
33+
/**
34+
* Shared instance returned while IO metrics are disabled. It discards all updates and never
35+
* registers anything with the metrics system.
36+
*/
37+
private static final MetricsIO DISABLED_INSTANCE =
38+
new MetricsIO(new NoOpMetricsIOSource(), new MetricsIOWrapperImpl());
39+
40+
private static volatile boolean enabled = false;
2841
private static volatile MetricsIO instance;
2942
private final MetricsIOSource source;
3043
private final MetricsIOWrapper wrapper;
@@ -41,21 +54,73 @@ public class MetricsIO {
4154
this.wrapper = wrapper;
4255
}
4356

57+
/**
58+
* Enables IO metrics for this process. Until this is called, {@link #getInstance()} returns an
59+
* instance that discards all updates, so processes that only read or write HFiles as a client
60+
* (bulk load, MapReduce tasks, command line tools) do not start the Hadoop metrics system or
61+
* register a "RegionServer" metrics source. Server processes call this during startup. The
62+
* metrics source itself is still created lazily, on first use after this call. See HBASE-30231
63+
* for more details.
64+
*/
65+
public static void enable() {
66+
enabled = true;
67+
}
68+
69+
/** Returns whether IO metrics have been enabled for this process. */
70+
public static boolean isEnabled() {
71+
return enabled;
72+
}
73+
74+
@RestrictedApi(explanation = "Should only be called in tests", link = "",
75+
allowedOnPath = ".*/src/test/.*")
76+
public static synchronized void resetForTesting() {
77+
enabled = false;
78+
instance = null;
79+
}
80+
4481
/**
4582
* Get a static instance for the MetricsIO so that accessors access the same instance. We want to
4683
* lazy initialize so that correct process name is in place. See HBASE-27966 for more details.
84+
* <p>
85+
* If IO metrics have not been enabled with {@link #enable()}, a shared instance that discards all
86+
* updates is returned. If the metrics source cannot be created, a no-op source is used instead,
87+
* so that a metrics failure never fails HFile reads or writes. See HBASE-30231 for more details.
4788
*/
4889
public static MetricsIO getInstance() {
90+
if (!enabled) {
91+
return DISABLED_INSTANCE;
92+
}
4993
if (instance == null) {
5094
synchronized (MetricsIO.class) {
5195
if (instance == null) {
52-
instance = new MetricsIO(new MetricsIOWrapperImpl());
96+
MetricsIOWrapper wrapper = new MetricsIOWrapperImpl();
97+
instance = createInstance(() -> new MetricsIO(wrapper), wrapper);
5398
}
5499
}
55100
}
56101
return instance;
57102
}
58103

104+
/**
105+
* Creates a {@link MetricsIO} with the given supplier, falling back to one backed by a no-op
106+
* {@link MetricsIOSource} if the supplier fails.
107+
* <p>
108+
* Registering the metrics source with the Hadoop metrics system reserves its name before the
109+
* registration has fully completed. If anything fails after that point, every later attempt to
110+
* register the same source fails with "Metrics source ... already exists!". Retrying would
111+
* therefore fail on every call for the rest of the JVM's life, so we fall back to a no-op source
112+
* once and never retry.
113+
*/
114+
static MetricsIO createInstance(Supplier<MetricsIO> supplier, MetricsIOWrapper wrapper) {
115+
try {
116+
return supplier.get();
117+
} catch (RuntimeException | LinkageError e) {
118+
LOG.warn("Failed to initialize the {} metrics source; FileSystem IO metrics will not be"
119+
+ " reported by this process", MetricsIOSource.METRICS_JMX_CONTEXT, e);
120+
return new MetricsIO(new NoOpMetricsIOSource(), wrapper);
121+
}
122+
}
123+
59124
public MetricsIOSource getMetricsSource() {
60125
return source;
61126
}
@@ -79,4 +144,75 @@ public void updateFsWriteTime(long t) {
79144
public void incrSlowFsRead() {
80145
source.incrSlowFsRead();
81146
}
147+
148+
/**
149+
* A {@link MetricsIOSource} that discards all updates. Used while IO metrics are disabled, or
150+
* when the real metrics source could not be created.
151+
*/
152+
static final class NoOpMetricsIOSource implements MetricsIOSource {
153+
154+
@Override
155+
public void updateFsReadTime(long t) {
156+
}
157+
158+
@Override
159+
public void updateFsPReadTime(long t) {
160+
}
161+
162+
@Override
163+
public void updateFsWriteTime(long t) {
164+
}
165+
166+
@Override
167+
public void incrSlowFsRead() {
168+
}
169+
170+
@Override
171+
public void init() {
172+
}
173+
174+
@Override
175+
public void setGauge(String gaugeName, long value) {
176+
}
177+
178+
@Override
179+
public void incGauge(String gaugeName, long delta) {
180+
}
181+
182+
@Override
183+
public void decGauge(String gaugeName, long delta) {
184+
}
185+
186+
@Override
187+
public void removeMetric(String key) {
188+
}
189+
190+
@Override
191+
public void incCounters(String counterName, long delta) {
192+
}
193+
194+
@Override
195+
public void updateHistogram(String name, long value) {
196+
}
197+
198+
@Override
199+
public String getMetricsContext() {
200+
return METRICS_CONTEXT;
201+
}
202+
203+
@Override
204+
public String getMetricsDescription() {
205+
return METRICS_DESCRIPTION;
206+
}
207+
208+
@Override
209+
public String getMetricsJmxContext() {
210+
return METRICS_JMX_CONTEXT;
211+
}
212+
213+
@Override
214+
public String getMetricsName() {
215+
return METRICS_NAME;
216+
}
217+
}
82218
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestMetricsIO.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,20 @@
1717
*/
1818
package org.apache.hadoop.hbase.io;
1919

20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertFalse;
22+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
23+
import static org.junit.jupiter.api.Assertions.assertNotSame;
24+
import static org.junit.jupiter.api.Assertions.assertSame;
25+
import static org.junit.jupiter.api.Assertions.assertTrue;
26+
2027
import org.apache.hadoop.hbase.CompatibilityFactory;
2128
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
2229
import org.apache.hadoop.hbase.testclassification.MiscTests;
2330
import org.apache.hadoop.hbase.testclassification.SmallTests;
31+
import org.apache.hadoop.metrics2.MetricsException;
32+
import org.junit.jupiter.api.AfterEach;
33+
import org.junit.jupiter.api.BeforeEach;
2434
import org.junit.jupiter.api.Tag;
2535
import org.junit.jupiter.api.Test;
2636

@@ -30,6 +40,45 @@ public class TestMetricsIO {
3040

3141
public MetricsAssertHelper HELPER = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
3242

43+
@BeforeEach
44+
public void setUp() {
45+
// Several tests register the same source, which the metrics system only allows in this mode.
46+
HELPER.init();
47+
MetricsIO.resetForTesting();
48+
}
49+
50+
@AfterEach
51+
public void tearDown() {
52+
MetricsIO.resetForTesting();
53+
}
54+
55+
@Test
56+
public void testDisabledByDefault() {
57+
assertFalse(MetricsIO.isEnabled());
58+
MetricsIO metrics = MetricsIO.getInstance();
59+
assertInstanceOf(MetricsIO.NoOpMetricsIOSource.class, metrics.getMetricsSource());
60+
assertSame(metrics, MetricsIO.getInstance());
61+
// Updates must be silently discarded while disabled.
62+
metrics.updateFsReadTime(100);
63+
metrics.updateFsPreadTime(200);
64+
metrics.updateFsWriteTime(300);
65+
metrics.incrSlowFsRead();
66+
}
67+
68+
@Test
69+
public void testEnable() {
70+
MetricsIO disabled = MetricsIO.getInstance();
71+
MetricsIO.enable();
72+
assertTrue(MetricsIO.isEnabled());
73+
MetricsIO enabled = MetricsIO.getInstance();
74+
assertNotSame(disabled, enabled);
75+
assertFalse(enabled.getMetricsSource() instanceof MetricsIO.NoOpMetricsIOSource);
76+
assertSame(enabled, MetricsIO.getInstance());
77+
78+
enabled.updateFsWriteTime(100);
79+
HELPER.assertCounter("fsWriteTime_numOps", 1, enabled.getMetricsSource());
80+
}
81+
3382
@Test
3483
public void testMetrics() {
3584
MetricsIO metrics = new MetricsIO(new MetricsIOWrapper() {
@@ -54,4 +103,42 @@ public long getChecksumFailures() {
54103
HELPER.assertCounter("fsPReadTime_numOps", 1, metrics.getMetricsSource());
55104
HELPER.assertCounter("fsWriteTime_numOps", 3, metrics.getMetricsSource());
56105
}
106+
107+
@Test
108+
public void testCreateInstanceUsesSupplierResult() {
109+
MetricsIOWrapper wrapper = () -> 0;
110+
MetricsIO expected = new MetricsIO(new MetricsIO.NoOpMetricsIOSource(), wrapper);
111+
assertSame(expected, MetricsIO.createInstance(() -> expected, wrapper));
112+
}
113+
114+
@Test
115+
public void testFallbackToNoOpSourceOnRuntimeException() {
116+
MetricsIOWrapper wrapper = () -> 0;
117+
MetricsIO metrics = MetricsIO.createInstance(() -> {
118+
// What a second registration attempt of the source fails with, see HBASE-30231
119+
throw new MetricsException("Metrics source RegionServer,sub=IO already exists!");
120+
}, wrapper);
121+
assertFallback(metrics, wrapper);
122+
}
123+
124+
@Test
125+
public void testFallbackToNoOpSourceOnLinkageError() {
126+
MetricsIOWrapper wrapper = () -> 0;
127+
MetricsIO metrics = MetricsIO.createInstance(() -> {
128+
throw new ExceptionInInitializerError("simulated static initializer failure");
129+
}, wrapper);
130+
assertFallback(metrics, wrapper);
131+
}
132+
133+
private static void assertFallback(MetricsIO metrics, MetricsIOWrapper wrapper) {
134+
assertInstanceOf(MetricsIO.NoOpMetricsIOSource.class, metrics.getMetricsSource());
135+
assertSame(wrapper, metrics.getWrapper());
136+
assertEquals(MetricsIOSource.METRICS_JMX_CONTEXT,
137+
metrics.getMetricsSource().getMetricsJmxContext());
138+
// Updates must be silently discarded rather than failing the caller's HFile read or write.
139+
metrics.updateFsReadTime(100);
140+
metrics.updateFsPreadTime(200);
141+
metrics.updateFsWriteTime(300);
142+
metrics.incrSlowFsRead();
143+
}
57144
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
2121
import static org.junit.jupiter.api.Assertions.assertFalse;
2222
import static org.junit.jupiter.api.Assertions.assertNull;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
2324

2425
import java.io.IOException;
2526
import org.apache.hadoop.conf.Configuration;
@@ -28,6 +29,7 @@
2829
import org.apache.hadoop.hbase.HConstants;
2930
import org.apache.hadoop.hbase.Waiter;
3031
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
32+
import org.apache.hadoop.hbase.io.MetricsIO;
3133
import org.apache.hadoop.hbase.testclassification.MasterTests;
3234
import org.apache.hadoop.hbase.testclassification.MediumTests;
3335
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -117,6 +119,24 @@ public void testStopDuringStart() throws IOException, KeeperException, Interrupt
117119
master.join();
118120
}
119121

122+
/**
123+
* The master reads and writes HFiles through its local MasterRegion, so it must enable IO metrics
124+
* even when no RegionServer runs in the same JVM. See HBASE-30231.
125+
*/
126+
@Test
127+
public void testMasterEnablesMetricsIO() throws IOException, InterruptedException {
128+
MetricsIO.resetForTesting();
129+
HMaster master = new HMaster(TESTUTIL.getConfiguration());
130+
master.start();
131+
try {
132+
assertTrue(MetricsIO.isEnabled());
133+
} finally {
134+
master.stopMaster();
135+
master.join();
136+
MetricsIO.resetForTesting();
137+
}
138+
}
139+
120140
@Test
121141
public void testMasterInitWithSameClientServerZKQuorum() throws Exception {
122142
Configuration conf = new Configuration(TESTUTIL.getConfiguration());

0 commit comments

Comments
 (0)