@Override public InternalScanner createScanner(ScanInfo scanInfo, List<StoreFileScanner> scanners, ScanType scanType, FileDetails fd, long smallestReadPoint) throws IOException { return new StoreScanner(store, scanInfo, scanners, scanType, smallestReadPoint, fd.earliestPutTs); } };
/** * @param store store * @param scanners Store file scanners. * @param scanType Scan type. * @param smallestReadPoint Smallest MVCC read point. * @param earliestPutTs Earliest put across all files. * @return A compaction scanner. */ protected InternalScanner createScanner(HStore store, ScanInfo scanInfo, List<StoreFileScanner> scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException { return new StoreScanner(store, scanInfo, scanners, scanType, smallestReadPoint, earliestPutTs); }
/** * @param store The store. * @param scanners Store file scanners. * @param smallestReadPoint Smallest MVCC read point. * @param earliestPutTs Earliest put across all files. * @param dropDeletesFromRow Drop deletes starting with this row, inclusive. Can be null. * @param dropDeletesToRow Drop deletes ending with this row, exclusive. Can be null. * @return A compaction scanner. */ protected InternalScanner createScanner(HStore store, ScanInfo scanInfo, List<StoreFileScanner> scanners, long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow, byte[] dropDeletesToRow) throws IOException { return new StoreScanner(store, scanInfo, scanners, smallestReadPoint, earliestPutTs, dropDeletesFromRow, dropDeletesToRow); } }
protected KeyValueScanner createScanner(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> targetCols, long readPt) throws IOException { return scan.isReversed() ? new ReversedStoreScanner(this, scanInfo, scan, targetCols, readPt) : new StoreScanner(this, scanInfo, scan, targetCols, readPt); }
/** * Creates a store scanner. * @param filesToCompact The files to be compacted. * @param scanType The scan type. * @return The store scanner. * @throws IOException if IO failure is encountered */ private StoreScanner createScanner(List<HStoreFile> filesToCompact, ScanType scanType) throws IOException { List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, true, false, false, HConstants.LATEST_TIMESTAMP); long ttl = HStore.determineTTLFromFamily(column); ScanInfo scanInfo = new ScanInfo(conf, column, ttl, 0, CellComparator.getInstance()); return new StoreScanner(scanInfo, scanType, scanners); }
@Test public void testScannerReseekDoesntNPE() throws Exception { List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { // Previously a updateReaders twice in a row would cause an NPE. In test this would also // normally cause an NPE because scan.store is null. So as long as we get through these // two calls we are good and the bug was quashed. scan.updateReaders(Collections.emptyList(), Collections.emptyList()); scan.updateReaders(Collections.emptyList(), Collections.emptyList()); scan.peek(); } }
@Test public void testSkipColumn() throws IOException { List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, getCols("a", "d"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); assertEquals(kvs[0], results.get(0)); assertEquals(kvs[3], results.get(1)); results.clear(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[kvs.length - 1], results.get(0)); results.clear(); assertEquals(false, scan.next(results)); } }
@Test public void testWildCardOneVersionScan() throws IOException { KeyValue [] kvs = new KeyValue [] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); assertEquals(kvs[0], results.get(0)); assertEquals(kvs[1], results.get(1)); } }
@Test public void testPreadNotEnabledForCompactionStoreScanners() throws Exception { long now = System.currentTimeMillis(); KeyValue[] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000, KeyValue.Type.DeleteFamily), create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); try (StoreScanner storeScanner = new StoreScanner(scanInfo, OptionalInt.empty(), ScanType.COMPACT_RETAIN_DELETES, scanners)) { assertFalse(storeScanner.isScanUsePread()); } } }
@Test public void testDeleteVersionSameTimestamp() throws IOException { KeyValue [] kvs = new KeyValue [] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertFalse(scan.next(results)); assertEquals(0, results.size()); } }
@Test public void testDeleteColumn() throws IOException { KeyValue [] kvs = new KeyValue[] { create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"), create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"), create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan(), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[3], results.get(0)); } }
@Test public void testScanSameTimestamp() throws IOException { // returns only 1 of these 2 even though same timestamp KeyValue [] kvs = new KeyValue[] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = Arrays.asList( new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), kvs) }); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); // this only uses maxVersions (default=1) and TimeRange (default=all) try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[0], results.get(0)); } }
@Test @Ignore("this fails, since we don't handle deletions, etc, in peek") public void testPeek() throws Exception { KeyValue[] kvs = new KeyValue [] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { assertNull(scan.peek()); } }
@Test public void testWontNextToNext() throws IOException { // build the scan file: KeyValue [] kvs = new KeyValue[] { create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); // this only uses maxVersions (default=1) and TimeRange (default=all) try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); scan.next(results); assertEquals(1, results.size()); assertEquals(kvs[0], results.get(0)); // should be ok... // now scan _next_ again. results.clear(); scan.next(results); assertEquals(1, results.size()); assertEquals(kvs[2], results.get(0)); results.clear(); scan.next(results); assertEquals(0, results.size()); } }
/** * Ensure that expired delete family markers don't override valid puts */ @Test public void testExpiredDeleteFamily() throws Exception { long now = System.currentTimeMillis(); KeyValue[] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, KeyValue.Type.DeleteFamily), create("R1", "cf", "a", now-10, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scan = new Scan(); scan.readVersions(1); // scanner with ttl equal to 500 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scanner.next(results)); assertEquals(1, results.size()); assertEquals(kvs[1], results.get(0)); results.clear(); assertEquals(false, scanner.next(results)); } }
@Test public void testDeleteVersionMaskingMultiplePuts() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); try (StoreScanner scan = new StoreScanner(new Scan().withStartRow(Bytes.toBytes("R1")), scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); // the two put at ts=now will be masked by the 1 delete, and // since the scan default returns 1 version we'll return the newest // key, which is kvs[2], now-100. assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs2[1], results.get(0)); } }
/** * Creates the scanner for flushing snapshot. Also calls coprocessors. * @param snapshotScanners * @param smallestReadPoint * @return The scanner; null if coprocessor is canceling the flush. */ protected final InternalScanner createScanner(List<KeyValueScanner> snapshotScanners, long smallestReadPoint, FlushLifeCycleTracker tracker) throws IOException { ScanInfo scanInfo; if (store.getCoprocessorHost() != null) { scanInfo = store.getCoprocessorHost().preFlushScannerOpen(store, tracker); } else { scanInfo = store.getScanInfo(); } InternalScanner scanner = new StoreScanner(store, scanInfo, snapshotScanners, ScanType.COMPACT_RETAIN_DELETES, smallestReadPoint, HConstants.OLDEST_TIMESTAMP); assert scanner != null; if (store.getCoprocessorHost() != null) { try { return store.getCoprocessorHost().preFlush(store, scanner, tracker); } catch (IOException ioe) { scanner.close(); throw ioe; } } return scanner; }
@Test public void testDeletedRowThenGoodRow() throws IOException { KeyValue [] kvs = new KeyValue [] { create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(0, results.size()); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[2], results.get(0)); assertEquals(false, scan.next(results)); } }
@Test public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2); try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(2, results.size()); assertEquals(kvs2[1], results.get(0)); assertEquals(kvs2[0], results.get(1)); } }
@Test public void testDeleteFamily() throws IOException { KeyValue[] kvs = new KeyValue[] { create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); try (StoreScanner scan = new StoreScanner(new Scan().readAllVersions(), scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scan.next(results)); assertEquals(0, results.size()); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[kvs.length - 1], results.get(0)); assertEquals(false, scan.next(results)); } }