Skip to content

Commit 4a6cb30

Browse files
anwar6953anwar6953
committed
TEPHRA-263 Fix test cases and TTL enforcement for tables configured to read non transactional data.
1 parent a346efe commit 4a6cb30

File tree

29 files changed

+100
-109
lines changed

29 files changed

+100
-109
lines changed

‎tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java

+8-3
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,14 @@ public static long getOldestVisibleTimestamp(long ttl, Transaction tx, boolean r
9494
if (ttl <= 0) {
9595
return 0;
9696
}
97-
long ttlFactor = readNonTxnData ? 1 : TxConstants.MAX_TX_PER_MS;
97+
long oldestVisibleTimestamp;
98+
if (readNonTxnData) {
99+
oldestVisibleTimestamp = getTimestamp(tx.getTransactionId()) - ttl;
100+
} else {
101+
oldestVisibleTimestamp = tx.getTransactionId() - ttl * TxConstants.MAX_TX_PER_MS;
102+
}
98103
// if the computed ttl is negative, return 0 because timestamps can not be negative
99-
return Math.max(0, tx.getTransactionId() - ttl * ttlFactor);
104+
return Math.max(0, oldestVisibleTimestamp);
100105
}
101106

102107
/**
@@ -119,7 +124,7 @@ public static long getMaxVisibleTimestamp(Transaction tx) {
119124
* as being written by this transaction (and therefore visible).
120125
*/
121126
public static Transaction createDummyTransaction(TransactionVisibilityState txVisibilityState) {
122-
return new Transaction(txVisibilityState.getReadPointer(), Long.MAX_VALUE,
127+
return new Transaction(txVisibilityState.getReadPointer(), txVisibilityState.getWritePointer(),
123128
Longs.toArray(txVisibilityState.getInvalid()),
124129
Longs.toArray(txVisibilityState.getInProgress().keySet()),
125130
TxUtils.getFirstShortInProgress(txVisibilityState.getInProgress()), TransactionType.SHORT);

‎tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java

+3-6
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ public class TransactionFilters {
4040
*/
4141
public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
4242
ScanType scanType) {
43-
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
44-
scanType, null));
43+
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
4544
}
4645

4746
/**
@@ -51,15 +50,13 @@ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlBy
5150
* @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
5251
* @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
5352
* these will be interpreted as "delete" markers and the column will be filtered out
54-
* @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
5553
* @param scanType the type of scan operation being performed
5654
* @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
5755
* calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
5856
* {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
5957
*/
6058
public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
61-
boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
62-
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
63-
scanType, cellFilter));
59+
ScanType scanType, @Nullable Filter cellFilter) {
60+
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
6461
}
6562
}

‎tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@ private boolean isRollbackOperation(OperationWithAttributes op) throws IOExcepti
446446
* @param type the type of scan operation being performed
447447
*/
448448
protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
449-
return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
449+
return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
450450
}
451451

452452
/**

‎tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java

+5-4
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public class TransactionVisibilityFilter extends FilterBase {
7070
*/
7171
public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
7272
ScanType scanType) {
73-
this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
73+
this(tx, ttlByFamily, allowEmptyValues, scanType, null);
7474
}
7575

7676
/**
@@ -80,20 +80,21 @@ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily
8080
* @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
8181
* @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
8282
* these will be interpreted as "delete" markers and the column will be filtered out
83-
* @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
8483
* @param scanType the type of scan operation being performed
8584
* @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
8685
* calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
8786
* {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
8887
*/
8988
public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
90-
boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
89+
ScanType scanType, @Nullable Filter cellFilter) {
9190
this.tx = tx;
9291
this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
9392
for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
9493
long familyTTL = ttlEntry.getValue();
9594
oldestTsByFamily.put(ttlEntry.getKey(),
96-
TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
95+
// we pass false for 'readNonTxData'. For non transactional data, we scale the cell timestamp
96+
// in #filterKeyValue, using TxUtils#getTimestampForTTL(long)
97+
TxUtils.getOldestVisibleTimestamp(familyTTL, tx, false));
9798
}
9899
this.allowEmptyValues = allowEmptyValues;
99100
this.clearDeletes =

‎tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/TransactionAwareHTableTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public class TransactionAwareHTableTest extends AbstractHBaseTableTest {
107107
public static TemporaryFolder tmpFolder = new TemporaryFolder();
108108

109109
private static MiniDFSCluster dfsCluster;
110-
110+
111111
private static final class TestBytes {
112112
private static final byte[] table = Bytes.toBytes("testtable");
113113
private static final byte[] family = Bytes.toBytes("f1");

‎tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
7474
TxFilterFactory txFilterFactory = new TxFilterFactory() {
7575
@Override
7676
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
77-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
77+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
7878
}
7979
};
8080
runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
9494
txFilterFactory = new TxFilterFactory() {
9595
@Override
9696
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
97-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
97+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
9898
}
9999
};
100100
runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
114114
txFilterFactory = new TxFilterFactory() {
115115
@Override
116116
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
117-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
117+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
118118
}
119119
};
120120
runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
134134
txFilterFactory = new TxFilterFactory() {
135135
@Override
136136
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
137-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
137+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
138138
}
139139
};
140140
runFilteringTest(txFilterFactory,
@@ -354,7 +354,7 @@ private interface TxFilterFactory {
354354
private class CustomTxFilter extends TransactionVisibilityFilter {
355355
public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
356356
@Nullable Filter cellFilter) {
357-
super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
357+
super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
358358
}
359359

360360
@Override

‎tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java

+3-6
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ public class TransactionFilters {
4040
*/
4141
public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
4242
ScanType scanType) {
43-
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
44-
scanType, null));
43+
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
4544
}
4645

4746
/**
@@ -51,15 +50,13 @@ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlBy
5150
* @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
5251
* @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
5352
* these will be interpreted as "delete" markers and the column will be filtered out
54-
* @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
5553
* @param scanType the type of scan operation being performed
5654
* @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
5755
* calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
5856
* {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
5957
*/
6058
public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
61-
boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
62-
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
63-
scanType, cellFilter));
59+
ScanType scanType, @Nullable Filter cellFilter) {
60+
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
6461
}
6562
}

‎tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@ private boolean isRollbackOperation(OperationWithAttributes op) throws IOExcepti
446446
* @param type the type of scan being performed
447447
*/
448448
protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
449-
return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
449+
return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
450450
}
451451

452452
/**

‎tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java

+5-4
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public class TransactionVisibilityFilter extends FilterBase {
7070
*/
7171
public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
7272
ScanType scanType) {
73-
this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
73+
this(tx, ttlByFamily, allowEmptyValues, scanType, null);
7474
}
7575

7676
/**
@@ -80,20 +80,21 @@ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily
8080
* @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
8181
* @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
8282
* these will be interpreted as "delete" markers and the column will be filtered out
83-
* @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
8483
* @param scanType the type of scan operation being performed
8584
* @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
8685
* calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
8786
* {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
8887
*/
8988
public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
90-
boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
89+
ScanType scanType, @Nullable Filter cellFilter) {
9190
this.tx = tx;
9291
this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
9392
for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
9493
long familyTTL = ttlEntry.getValue();
9594
oldestTsByFamily.put(ttlEntry.getKey(),
96-
TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
95+
// we pass false for 'readNonTxData'. For non transactional data, we scale the cell timestamp
96+
// in #filterKeyValue, using TxUtils#getTimestampForTTL(long)
97+
TxUtils.getOldestVisibleTimestamp(familyTTL, tx, false));
9798
}
9899
this.allowEmptyValues = allowEmptyValues;
99100
this.clearDeletes =

‎tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
7474
TxFilterFactory txFilterFactory = new TxFilterFactory() {
7575
@Override
7676
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
77-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
77+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
7878
}
7979
};
8080
runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
9494
txFilterFactory = new TxFilterFactory() {
9595
@Override
9696
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
97-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
97+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
9898
}
9999
};
100100
runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
114114
txFilterFactory = new TxFilterFactory() {
115115
@Override
116116
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
117-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
117+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
118118
}
119119
};
120120
runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
134134
txFilterFactory = new TxFilterFactory() {
135135
@Override
136136
public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
137-
return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
137+
return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
138138
}
139139
};
140140
runFilteringTest(txFilterFactory,
@@ -353,7 +353,7 @@ private interface TxFilterFactory {
353353
private class CustomTxFilter extends TransactionVisibilityFilter {
354354
public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
355355
@Nullable Filter cellFilter) {
356-
super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
356+
super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
357357
}
358358

359359
@Override

‎tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java

+3-6
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ public class TransactionFilters {
4040
*/
4141
public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
4242
ScanType scanType) {
43-
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
44-
scanType, null));
43+
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
4544
}
4645

4746
/**
@@ -51,15 +50,13 @@ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlBy
5150
* @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
5251
* @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
5352
* these will be interpreted as "delete" markers and the column will be filtered out
54-
* @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
5553
* @param scanType the type of scan operation being performed
5654
* @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
5755
* calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
5856
* {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
5957
*/
6058
public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
61-
boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
62-
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
63-
scanType, cellFilter));
59+
ScanType scanType, @Nullable Filter cellFilter) {
60+
return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
6461
}
6562
}

‎tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@ private boolean isRollbackOperation(OperationWithAttributes op) throws IOExcepti
446446
* @param type the type of scan being performed
447447
*/
448448
protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
449-
return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
449+
return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
450450
}
451451

452452
/**

0 commit comments

Comments
 (0)