@Override Path getPath() { return file.getPath(); } }
@Override public String apply(HStoreFile sf) { return sf.getPath().getName(); } }
/** * Gets the file name. * @return The file name. */ public String getFileName() { return sf.getPath().getName(); }
@Override public String getName() { return file.getPath().getName(); }
@Override public boolean isBulkLoadResult() { boolean bulkLoadedHFile = false; String fileName = this.getPath().getName(); int startPos = fileName.indexOf("SeqId_"); if (startPos != -1) { bulkLoadedHFile = true; } return bulkLoadedHFile || (metadataMap != null && metadataMap.containsKey(BULKLOAD_TIME_KEY)); }
/** * Creates a deterministic hash code for store file collection. */ public static OptionalInt getDeterministicRandomSeed(Collection<HStoreFile> files) { return files.stream().mapToInt(f -> f.getPath().getName().hashCode()).findFirst(); }
/** * Closes the readers of store files. * @param storeFiles The store files to be closed. */ private void closeStoreFileReaders(List<HStoreFile> storeFiles) { for (HStoreFile storeFile : storeFiles) { try { storeFile.closeStoreFile(true); } catch (IOException e) { LOG.warn("Failed to close the reader on store file " + storeFile.getPath(), e); } } }
/** * Delete this file * @throws IOException */ public void deleteStoreFile() throws IOException { boolean evictOnClose = cacheConf != null ? cacheConf.shouldEvictOnClose() : true; closeStoreFile(evictOnClose); this.fs.delete(getPath(), true); }
@Override public Collection<HStoreFile> getUnneededFiles(long maxTs, List<HStoreFile> filesCompacting) { ImmutableList<HStoreFile> files = storefiles; // 1) We can never get rid of the last file which has the maximum seqid. // 2) Files that are not the latest can't become one due to (1), so the rest are fair game. return files.stream().limit(Math.max(0, files.size() - 1)).filter(sf -> { long fileTs = sf.getReader().getMaxTimestamp(); if (fileTs < maxTs && !filesCompacting.contains(sf)) { LOG.info("Found an expired store file {} whose maxTimestamp is {}, which is below {}", sf.getPath(), fileTs, maxTs); return true; } else { return false; } }).collect(Collectors.toList()); }
/** * @return Map of StoreFiles by column family */ private NavigableMap<byte[], List<Path>> getStoreFiles() { NavigableMap<byte[], List<Path>> allStoreFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR); for (HStore store : stores.values()) { Collection<HStoreFile> storeFiles = store.getStorefiles(); if (storeFiles == null) { continue; } List<Path> storeFileNames = new ArrayList<>(); for (HStoreFile storeFile : storeFiles) { storeFileNames.add(storeFile.getPath()); } allStoreFiles.put(store.getColumnFamilyDescriptor().getName(), storeFileNames); } return allStoreFiles; }
private Collection<HStoreFile> findExpiredFiles(ImmutableList<HStoreFile> stripe, long maxTs, List<HStoreFile> filesCompacting, Collection<HStoreFile> expiredStoreFiles) { // Order by seqnum is reversed. for (int i = 1; i < stripe.size(); ++i) { HStoreFile sf = stripe.get(i); synchronized (sf) { long fileTs = sf.getReader().getMaxTimestamp(); if (fileTs < maxTs && !filesCompacting.contains(sf)) { LOG.info("Found an expired store file: " + sf.getPath() + " whose maxTimestamp is " + fileTs + ", which is below " + maxTs); if (expiredStoreFiles == null) { expiredStoreFiles = new ArrayList<>(); } expiredStoreFiles.add(sf); } } } return expiredStoreFiles; }
private static HStoreFile createFile() throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getPath()).thenReturn(new Path("file")); StoreFileReader r = mock(StoreFileReader.class); when(r.length()).thenReturn(10L); when(sf.getReader()).thenReturn(r); return sf; }
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); } final byte[] splitRow = getSplitRow(); final String familyName = Bytes.toString(family); final Path path_first = regionFs.splitStoreFile(this.daughter_1_RI, familyName, sf, splitRow, false, splitPolicy); final Path path_second = regionFs.splitStoreFile(this.daughter_2_RI, familyName, sf, splitRow, true, splitPolicy); if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); } return new Pair<Path,Path>(path_first, path_second); }
private static HStoreFile createFile() throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getMetadataValue(any())) .thenReturn(StripeStoreFileManager.INVALID_KEY); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.getPath()).thenReturn(new Path("moo")); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
private static HStoreFile createFile(long size) throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getPath()).thenReturn(new Path("moo")); StoreFileReader r = mock(StoreFileReader.class); when(r.getEntries()).thenReturn(size); when(r.length()).thenReturn(size); when(r.getBloomFilterType()).thenReturn(BloomType.NONE); when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class)); when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class)); when(sf.getReader()).thenReturn(r); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
private static List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(htd.getTableName())) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private static List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region: TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegions(htd.getTableName())) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private static List<Path> findCompactedStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { Collection<HStoreFile> compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (compactedfiles != null) { for (HStoreFile storefile : compactedfiles) { paths.add(storefile.getPath()); } } } } return paths; }
private void addStoreFile() throws IOException { HStoreFile f = this.store.getStorefiles().iterator().next(); Path storedir = f.getPath().getParent(); long seqid = this.store.getMaxSequenceId().orElse(0L); Configuration c = TEST_UTIL.getConfiguration(); FileSystem fs = FileSystem.get(c); HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build(); StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), fs) .withOutputDir(storedir) .withFileContext(fileContext) .build(); w.appendMetadata(seqid + 1, false); w.close(); LOG.info("Added store file:" + w.getPath()); }