public HDFSBlocksDistribution getHDFSBlocksDistribution() { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); stores.values().stream().filter(s -> s.getStorefiles() != null) .flatMap(s -> s.getStorefiles().stream()).map(HStoreFile::getHDFSBlockDistribution) .forEachOrdered(hdfsBlocksDistribution::add); return hdfsBlocksDistribution; }
@Override public OptionalLong getMaxSequenceId() { return StoreUtils.getMaxSequenceIdInList(this.getStorefiles()); }
@Override public OptionalLong getMaxMemStoreTS() { return StoreUtils.getMaxMemStoreTSInList(this.getStorefiles()); }
private void logRegionFiles() { if (LOG.isTraceEnabled()) { LOG.trace(getRegionInfo().getEncodedName() + " : Store files for region: "); stores.values().stream().filter(s -> s.getStorefiles() != null) .flatMap(s -> s.getStorefiles().stream()) .forEachOrdered(sf -> LOG.trace(getRegionInfo().getEncodedName() + " : " + sf)); } }
@Override public boolean hasReferences() { // Grab the read lock here, because we need to ensure that: only when the atomic // replaceStoreFiles(..) finished, we can get all the complete store file list. this.lock.readLock().lock(); try { // Merge the current store files with compacted files here due to HBASE-20940. Collection<HStoreFile> allStoreFiles = new ArrayList<>(getStorefiles()); allStoreFiles.addAll(getCompactedFiles()); return StoreUtils.hasReferences(allStoreFiles); } finally { this.lock.readLock().unlock(); } }
@Override public List<String> getStoreFileList(byte[][] columns) throws IllegalArgumentException { List<String> storeFileNames = new ArrayList<>(); synchronized (closeLock) { for (byte[] column : columns) { HStore store = this.stores.get(column); if (store == null) { throw new IllegalArgumentException( "No column family : " + new String(column, StandardCharsets.UTF_8) + " available"); } Collection<HStoreFile> storeFiles = store.getStorefiles(); if (storeFiles == null) { continue; } for (HStoreFile storeFile : storeFiles) { storeFileNames.add(storeFile.getPath().toString()); } logRegionFiles(); } } return storeFileNames; }
/** * @return Map of StoreFiles by column family */ private NavigableMap<byte[], List<Path>> getStoreFiles() { NavigableMap<byte[], List<Path>> allStoreFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR); for (HStore store : stores.values()) { Collection<HStoreFile> storeFiles = store.getStorefiles(); if (storeFiles == null) { continue; } List<Path> storeFileNames = new ArrayList<>(); for (HStoreFile storeFile : storeFiles) { storeFileNames.add(storeFile.getPath()); } allStoreFiles.put(store.getColumnFamilyDescriptor().getName(), storeFileNames); } return allStoreFiles; }
private Set<StoreFileReader> getStreamReaders() { List<HStore> stores = REGION.getStores(); Assert.assertEquals(1, stores.size()); HStore firstStore = stores.get(0); Assert.assertNotNull(firstStore); Collection<HStoreFile> storeFiles = firstStore.getStorefiles(); Assert.assertEquals(1, storeFiles.size()); HStoreFile firstSToreFile = storeFiles.iterator().next(); Assert.assertNotNull(firstSToreFile); return Collections.unmodifiableSet(firstSToreFile.streamReaders); }
private void archiveStoreFile(int index) throws IOException { Collection<HStoreFile> files = this.store.getStorefiles(); HStoreFile sf = null; Iterator<HStoreFile> it = files.iterator(); for (int i = 0; i <= index; i++) { sf = it.next(); } store.getRegionFileSystem().removeStoreFiles(store.getColumnFamilyName(), Lists.newArrayList(sf)); }
private static List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(htd.getTableName())) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private static List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region: TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegions(htd.getTableName())) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private int count() throws IOException { int count = 0; for (HStoreFile f: this.r.stores. get(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
private void flush(int storeFilessize) throws IOException{ this.store.snapshot(); flushStore(store, id++); assertEquals(storeFilessize, this.store.getStorefiles().size()); assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); }
@Test public void testFlushedFileWithNoTags() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor(fam1)); HRegionInfo info = new HRegionInfo(tableName, null, null, false); Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); region = HBaseTestingUtility.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), htd); Put put = new Put(Bytes.toBytes("a-b-0-0")); put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); region.put(put); region.flush(true); HStore store = region.getStore(fam1); Collection<HStoreFile> storefiles = store.getStorefiles(); for (HStoreFile sf : storefiles) { assertFalse("Tags should not be present " ,sf.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }
private int count() throws IOException { int count = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
private void addStoreFile() throws IOException { HStoreFile f = this.store.getStorefiles().iterator().next(); Path storedir = f.getPath().getParent(); long seqid = this.store.getMaxSequenceId().orElse(0L); Configuration c = TEST_UTIL.getConfiguration(); FileSystem fs = FileSystem.get(c); HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build(); StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), fs) .withOutputDir(storedir) .withFileContext(fileContext) .build(); w.appendMetadata(seqid + 1, false); w.close(); LOG.info("Added store file:" + w.getPath()); }
private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException { init(this.name.getMethodName(), TEST_UTIL.getConfiguration(), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build()); long currentTs = 100; long minTs = currentTs; // the extra cell won't be flushed to disk, // so the min of timerange will be different between memStore and hfile. for (int i = 0; i != (maxVersion + 1); ++i) { this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[])null), null); if (i == 1) { minTs = currentTs; } } flushStore(store, id++); Collection<HStoreFile> files = store.getStorefiles(); assertEquals(1, files.size()); HStoreFile f = files.iterator().next(); f.initReader(); StoreFileReader reader = f.getReader(); assertEquals(minTs, reader.timeRange.getMin()); assertEquals(currentTs, reader.timeRange.getMax()); }
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte [] row = CellUtil.cloneRow(scanner.getCell()); if (Bytes.equals(row, STARTROW)) { count1++; } else if(Bytes.equals(row, secondRowBytes)) { count2++; } } while(scanner.next()); } assertEquals(countRow1,count1); assertEquals(countRow2,count2); }