Skip to content

Commit 66316c7

Browse files
xiaguangleixiaguanglei02
authored and
ndimiduk
committed
HBASE-29158 Unknown checksum type code exception occurred while reading HFileBlock (#6740)
Co-authored-by: xiaguanglei <[email protected]> Signed-off-by: Nick Dimiduk <[email protected]>
1 parent ab1bdd3 commit 66316c7

File tree

2 files changed

+183
-1
lines changed

2 files changed

+183
-1
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java

+44
Original file line numberDiff line numberDiff line change
@@ -1587,6 +1587,24 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, boolean
15871587
}
15881588
}
15891589

1590+
/**
1591+
* Check that checksumType on {@code headerBuf} read from a block header seems reasonable,
1592+
* within the known value range.
1593+
* @return {@code true} if the headerBuf is safe to proceed, {@code false} otherwise.
1594+
*/
1595+
private boolean checkCheckSumTypeOnHeaderBuf(ByteBuff headerBuf) {
1596+
if (headerBuf == null) {
1597+
return true;
1598+
}
1599+
byte b = headerBuf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX);
1600+
for (ChecksumType t : ChecksumType.values()) {
1601+
if (t.getCode() == b) {
1602+
return true;
1603+
}
1604+
}
1605+
return false;
1606+
}
1607+
15901608
/**
15911609
* Check that {@code value} read from a block header seems reasonable, within a large margin of
15921610
* error.
@@ -1742,6 +1760,21 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
17421760
onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf, checksumSupport);
17431761
}
17441762

1763+
// Inspect the header's checksumType for known valid values. If we don't find such a value,
1764+
// assume that the bytes read are corrupted.We will clear the cached value and roll back to
1765+
// HDFS checksum
1766+
if (!checkCheckSumTypeOnHeaderBuf(headerBuf)) {
1767+
if (verifyChecksum) {
1768+
invalidateNextBlockHeader();
1769+
span.addEvent("Falling back to HDFS checksumming.", attributesBuilder.build());
1770+
return null;
1771+
} else {
1772+
throw new IOException(
1773+
"Unknown checksum type code " + headerBuf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX)
1774+
+ "for file " + pathName + ", the headerBuf of HFileBlock may corrupted.");
1775+
}
1776+
}
1777+
17451778
// The common case is that onDiskSizeWithHeader was produced by a read without checksum
17461779
// validation, so give it a sanity check before trying to use it.
17471780
if (!checkOnDiskSizeWithHeader(onDiskSizeWithHeader)) {
@@ -1885,6 +1918,17 @@ private boolean validateChecksum(long offset, ByteBuff data, int hdrSize) {
18851918
if (!fileContext.isUseHBaseChecksum()) {
18861919
return false;
18871920
}
1921+
1922+
// If the checksumType of the read block header is incorrect, it indicates that the block is
1923+
// corrupted and can be directly rolled back to HDFS checksum verification
1924+
if (!checkCheckSumTypeOnHeaderBuf(data)) {
1925+
HFile.LOG.warn(
1926+
"HBase checksumType verification failed for file {} at offset {} filesize {}"
1927+
+ " checksumType {}. Retrying read with HDFS checksums turned on...",
1928+
pathName, offset, fileSize, data.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
1929+
return false;
1930+
}
1931+
18881932
return ChecksumUtil.validateChecksum(data, pathName, offset, hdrSize);
18891933
}
18901934

‎hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockHeaderCorruption.java

+139-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import static org.hamcrest.Matchers.instanceOf;
2424
import static org.hamcrest.Matchers.startsWith;
2525
import static org.junit.Assert.assertEquals;
26+
import static org.junit.Assert.assertNotEquals;
2627
import static org.junit.Assert.assertTrue;
2728
import static org.junit.Assert.fail;
2829

@@ -55,6 +56,7 @@
5556
import org.apache.hadoop.hbase.testclassification.IOTests;
5657
import org.apache.hadoop.hbase.testclassification.SmallTests;
5758
import org.apache.hadoop.hbase.util.Bytes;
59+
import org.apache.hadoop.hbase.util.ChecksumType;
5860
import org.hamcrest.Description;
5961
import org.hamcrest.Matcher;
6062
import org.hamcrest.TypeSafeMatcher;
@@ -93,6 +95,141 @@ public TestHFileBlockHeaderCorruption() throws IOException {
9395
ruleChain = RuleChain.outerRule(testName).around(hFileTestRule);
9496
}
9597

98+
@Test
99+
public void testChecksumTypeCorruptionFirstBlock() throws Exception {
100+
HFileBlockChannelPosition firstBlock = null;
101+
try {
102+
try (HFileBlockChannelPositionIterator it =
103+
new HFileBlockChannelPositionIterator(hFileTestRule)) {
104+
assertTrue(it.hasNext());
105+
firstBlock = it.next();
106+
}
107+
108+
Corrupter c = new Corrupter(firstBlock);
109+
110+
logHeader(firstBlock);
111+
112+
// test corrupted HFileBlock with unknown checksumType code -1
113+
c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { -1 }));
114+
logHeader(firstBlock);
115+
try (HFileBlockChannelPositionIterator it =
116+
new HFileBlockChannelPositionIterator(hFileTestRule)) {
117+
CountingConsumer consumer = new CountingConsumer(it);
118+
try {
119+
consumer.readFully();
120+
fail();
121+
} catch (Exception e) {
122+
assertThat(e, new IsThrowableMatching().withInstanceOf(IOException.class)
123+
.withMessage(startsWith("Unknown checksum type code")));
124+
}
125+
assertEquals(0, consumer.getItemsRead());
126+
}
127+
128+
// valid checksumType code test
129+
for (ChecksumType t : ChecksumType.values()) {
130+
testValidChecksumTypeReadBlock(t.getCode(), c, firstBlock);
131+
}
132+
133+
c.restore();
134+
// test corrupted HFileBlock with unknown checksumType code 3
135+
c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { 3 }));
136+
logHeader(firstBlock);
137+
try (HFileBlockChannelPositionIterator it =
138+
new HFileBlockChannelPositionIterator(hFileTestRule)) {
139+
CountingConsumer consumer = new CountingConsumer(it);
140+
try {
141+
consumer.readFully();
142+
fail();
143+
} catch (Exception e) {
144+
assertThat(e, new IsThrowableMatching().withInstanceOf(IOException.class)
145+
.withMessage(startsWith("Unknown checksum type code")));
146+
}
147+
assertEquals(0, consumer.getItemsRead());
148+
}
149+
} finally {
150+
if (firstBlock != null) {
151+
firstBlock.close();
152+
}
153+
}
154+
}
155+
156+
@Test
157+
public void testChecksumTypeCorruptionSecondBlock() throws Exception {
158+
HFileBlockChannelPosition secondBlock = null;
159+
try {
160+
try (HFileBlockChannelPositionIterator it =
161+
new HFileBlockChannelPositionIterator(hFileTestRule)) {
162+
assertTrue(it.hasNext());
163+
it.next();
164+
assertTrue(it.hasNext());
165+
secondBlock = it.next();
166+
}
167+
168+
Corrupter c = new Corrupter(secondBlock);
169+
170+
logHeader(secondBlock);
171+
// test corrupted HFileBlock with unknown checksumType code -1
172+
c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { -1 }));
173+
logHeader(secondBlock);
174+
try (HFileBlockChannelPositionIterator it =
175+
new HFileBlockChannelPositionIterator(hFileTestRule)) {
176+
CountingConsumer consumer = new CountingConsumer(it);
177+
try {
178+
consumer.readFully();
179+
fail();
180+
} catch (Exception e) {
181+
assertThat(e, new IsThrowableMatching().withInstanceOf(RuntimeException.class)
182+
.withMessage(startsWith("Unknown checksum type code")));
183+
}
184+
assertEquals(1, consumer.getItemsRead());
185+
}
186+
187+
// valid checksumType code test
188+
for (ChecksumType t : ChecksumType.values()) {
189+
testValidChecksumTypeReadBlock(t.getCode(), c, secondBlock);
190+
}
191+
192+
c.restore();
193+
// test corrupted HFileBlock with unknown checksumType code 3
194+
c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { 3 }));
195+
logHeader(secondBlock);
196+
try (HFileBlockChannelPositionIterator it =
197+
new HFileBlockChannelPositionIterator(hFileTestRule)) {
198+
CountingConsumer consumer = new CountingConsumer(it);
199+
try {
200+
consumer.readFully();
201+
fail();
202+
} catch (Exception e) {
203+
assertThat(e, new IsThrowableMatching().withInstanceOf(RuntimeException.class)
204+
.withMessage(startsWith("Unknown checksum type code")));
205+
}
206+
assertEquals(1, consumer.getItemsRead());
207+
}
208+
} finally {
209+
if (secondBlock != null) {
210+
secondBlock.close();
211+
}
212+
}
213+
}
214+
215+
public void testValidChecksumTypeReadBlock(byte checksumTypeCode, Corrupter c,
216+
HFileBlockChannelPosition testBlock) throws IOException {
217+
c.restore();
218+
c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX,
219+
ByteBuffer.wrap(new byte[] { checksumTypeCode }));
220+
logHeader(testBlock);
221+
try (
222+
HFileBlockChannelPositionIterator it = new HFileBlockChannelPositionIterator(hFileTestRule)) {
223+
CountingConsumer consumer = new CountingConsumer(it);
224+
try {
225+
consumer.readFully();
226+
} catch (Exception e) {
227+
fail("test fail: valid checksumType are not executing properly");
228+
}
229+
assertNotEquals(0, consumer.getItemsRead());
230+
}
231+
}
232+
96233
@Test
97234
public void testOnDiskSizeWithoutHeaderCorruptionFirstBlock() throws Exception {
98235
HFileBlockChannelPosition firstBlock = null;
@@ -331,7 +468,8 @@ public HFileBlockChannelPositionIterator(HFileTestRule hFileTestRule) throws IOE
331468
try {
332469
reader = HFile.createReader(hfs, hfsPath, CacheConfig.DISABLED, true, conf);
333470
HFileBlock.FSReader fsreader = reader.getUncachedBlockReader();
334-
iter = fsreader.blockRange(0, hfs.getFileStatus(hfsPath).getLen());
471+
// The read block offset cannot out of the range:0,loadOnOpenDataOffset
472+
iter = fsreader.blockRange(0, reader.getTrailer().getLoadOnOpenDataOffset());
335473
} catch (IOException e) {
336474
if (reader != null) {
337475
closeQuietly(reader::close);

0 commit comments

Comments
 (0)