@Test public void testColumns() throws IOException { TreeSet<byte[]> trackedColumns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); trackedColumns.add(col1); trackedColumns.add(col3); NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(trackedColumns, comparator, 1, 3, 3, 10000); KeyValue keyValue = new KeyValue(row, family, col0, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.INCLUDE, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col3, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.INCLUDE, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col4, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_ROW, tracker.checkColumn(keyValue, keyValue.getTypeByte())); }
@Override public MatchCode checkColumn(Cell cell, byte type) throws IOException { if (columns == null) { return MatchCode.INCLUDE; } while (!done()) { int c = CellUtil.compareQualifiers(cell, columns[columnIndex], 0, columns[columnIndex].length); if (c < 0) { return MatchCode.SEEK_NEXT_COL; } if (c == 0) { // We drop old version in #isDeleted, so here we must return INCLUDE. return MatchCode.INCLUDE; } columnIndex++; } // No more columns left, we are done with this query return MatchCode.SEEK_NEXT_ROW; }
long duplicateMvcc = prepare(cell);
@Test public void testVersionsDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); KeyValue delete = new KeyValue(row, family, col1, 20000, KeyValue.Type.DeleteColumn, value); delete.setSequenceId(1000); delete.setTimestamp(20000); tracker.add(delete); put.setSequenceId(1001); put.setTimestamp(19999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); delete = new KeyValue(row, family, col2, 20000, KeyValue.Type.DeleteColumn, value); delete.setSequenceId(1002); delete.setTimestamp(20000); tracker.add(delete); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(19999); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); }
@Test public void testMinVersionsAndTTL() throws IOException { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 30000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.INCLUDE_AND_SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19999); keyValue.setSequenceId(999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals( MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19999); keyValue.setSequenceId(998); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19998); keyValue.setSequenceId(997); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19997); keyValue.setSequenceId(996); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false));
@Test public void testMaxVersionMask() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(998); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19998); keyValue.setSequenceId(997); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19997); keyValue.setSequenceId(996); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(1002); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(1001); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19998); keyValue.setSequenceId(1003); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19997);
deleteTracker = new NewVersionBehaviorTracker(columns, scanInfo.getComparator(), scanInfo.getMinVersions(), scanInfo.getMaxVersions(), resultMaxVersion, oldestUnexpiredTS);
/** * Note maxVersion and minVersion must set according to cf's conf, not user's scan parameter. * * @param columns columns specified user in query * @param comparartor the cell comparator * @param minVersion The minimum number of versions to keep(used when TTL is set). * @param maxVersion The maximum number of versions in CF's conf * @param resultMaxVersions maximum versions to return per column, which may be different from * maxVersion * @param oldestUnexpiredTS the oldest timestamp we are interested in, based on TTL */ public NewVersionBehaviorTracker(NavigableSet<byte[]> columns, CellComparator comparartor, int minVersion, int maxVersion, int resultMaxVersions, long oldestUnexpiredTS) { this.maxVersions = maxVersion; this.minVersions = minVersion; this.resultMaxVersions = resultMaxVersions; this.oldestStamp = oldestUnexpiredTS; if (columns != null && columns.size() > 0) { this.columns = new byte[columns.size()][]; int i = 0; for (byte[] column : columns) { this.columns[i++] = column; } } this.comparator = comparartor; reset(); }
@Override public void reset() { delColMap.clear(); delFamMap.clear(); lastCqArray = null; lastCqLength = 0; lastCqOffset = 0; lastCqTs = Long.MIN_VALUE; lastCqMvcc = 0; lastCqType = 0; columnIndex = 0; countCurrentCol = 0; resetInternal(); }
@Test public void testFamilyVersionsDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue delete = new KeyValue(row, family, null, 20000, KeyValue.Type.DeleteFamily, value); delete.setSequenceId(1000); delete.setTimestamp(20000); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); tracker.add(delete); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(998); put.setTimestamp(19999); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); }
@Test public void testMinVersionsAndTTL() throws IOException { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 30000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.INCLUDE_AND_SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19999); keyValue.setSequenceId(999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals( MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19999); keyValue.setSequenceId(998); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19998); keyValue.setSequenceId(997); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false)); keyValue.setTimestamp(19997); keyValue.setSequenceId(996); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkVersions(keyValue, keyValue.getTimestamp(), keyValue.getTypeByte(), false));
@Test public void testMaxVersionMask() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(998); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19998); keyValue.setSequenceId(997); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19997); keyValue.setSequenceId(996); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(1002); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19999); keyValue.setSequenceId(1001); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19998); keyValue.setSequenceId(1003); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); keyValue.setTimestamp(19997);
@Test public void testFamilyVersionDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue delete = new KeyValue(row, family, null, 20000, KeyValue.Type.DeleteFamilyVersion, value); delete.setSequenceId(1000); delete.setTimestamp(20000); tracker.add(delete); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put.setSequenceId(1002); put.setTimestamp(19999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(998); put.setTimestamp(19999); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(put)); }
@Test public void testColumns() throws IOException { TreeSet<byte[]> trackedColumns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); trackedColumns.add(col1); trackedColumns.add(col3); NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(trackedColumns, comparator, 1, 3, 3, 10000); KeyValue keyValue = new KeyValue(row, family, col0, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.INCLUDE, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_COL, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col3, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.INCLUDE, tracker.checkColumn(keyValue, keyValue.getTypeByte())); keyValue = new KeyValue(row, family, col4, 20000, KeyValue.Type.Put, value); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(keyValue)); assertEquals(MatchCode.SEEK_NEXT_ROW, tracker.checkColumn(keyValue, keyValue.getTypeByte())); }
@Override public void add(Cell cell) { prepare(cell); byte type = cell.getTypeByte(); switch (Type.codeToType(type)) { // By the order of seen. We put null cq at first. case DeleteFamily: // Delete all versions of all columns of the specified family delFamMap.put(cell.getSequenceId(), new DeleteVersionsNode(cell.getTimestamp(), cell.getSequenceId())); break; case DeleteFamilyVersion: // Delete all columns of the specified family and specified version delFamMap.ceilingEntry(cell.getSequenceId()).getValue().addVersionDelete(cell); break; // These two kinds of markers are mix with Puts. case DeleteColumn: // Delete all versions of the specified column delColMap.put(cell.getSequenceId(), new DeleteVersionsNode(cell.getTimestamp(), cell.getSequenceId())); break; case Delete: // Delete the specified version of the specified column. delColMap.ceilingEntry(cell.getSequenceId()).getValue().addVersionDelete(cell); break; default: throw new AssertionError("Unknown delete marker type for " + cell); } }
@Test public void testVersionDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); KeyValue delete = new KeyValue(row, family, col1, 20000, KeyValue.Type.Delete, value); delete.setSequenceId(1000); delete.setTimestamp(20000); tracker.add(delete); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); tracker.add(delete); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put.setSequenceId(1002); put.setTimestamp(19999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(998); put.setTimestamp(19999); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(put));
@Test public void testFamilyVersionsDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue delete = new KeyValue(row, family, null, 20000, KeyValue.Type.DeleteFamily, value); delete.setSequenceId(1000); delete.setTimestamp(20000); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); tracker.add(delete); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(998); put.setTimestamp(19999); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); }
@Test public void testVersionsDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); KeyValue delete = new KeyValue(row, family, col1, 20000, KeyValue.Type.DeleteColumn, value); delete.setSequenceId(1000); delete.setTimestamp(20000); tracker.add(delete); put.setSequenceId(1001); put.setTimestamp(19999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); delete = new KeyValue(row, family, col2, 20000, KeyValue.Type.DeleteColumn, value); delete.setSequenceId(1002); delete.setTimestamp(20000); tracker.add(delete); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(19999); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(19998); assertEquals(DeleteResult.COLUMN_DELETED, tracker.isDeleted(put)); }
@Test public void testFamilyVersionDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue delete = new KeyValue(row, family, null, 20000, KeyValue.Type.DeleteFamilyVersion, value); delete.setSequenceId(1000); delete.setTimestamp(20000); tracker.add(delete); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put.setSequenceId(1002); put.setTimestamp(19999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(998); put.setTimestamp(19999); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(put)); }
@Test public void testVersionDelete() { NewVersionBehaviorTracker tracker = new NewVersionBehaviorTracker(null, comparator, 1, 3, 3, 10000); KeyValue put = new KeyValue(row, family, col1, 20000, KeyValue.Type.Put, value); KeyValue delete = new KeyValue(row, family, col1, 20000, KeyValue.Type.Delete, value); delete.setSequenceId(1000); delete.setTimestamp(20000); tracker.add(delete); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); tracker.add(delete); put = new KeyValue(row, family, col2, 20000, KeyValue.Type.Put, value); put.setSequenceId(1001); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put.setSequenceId(999); put.setTimestamp(20000); assertEquals(DeleteResult.VERSION_DELETED, tracker.isDeleted(put)); put.setSequenceId(1002); put.setTimestamp(19999); assertEquals(DeleteResult.NOT_DELETED, tracker.isDeleted(put)); put.setSequenceId(998); put.setTimestamp(19999); assertEquals(DeleteResult.VERSION_MASKED, tracker.isDeleted(put));