/** * Gets a component sequence based on its unique timestamp. * e.g. a component file 1_3_b * will return a component sequence 1_3 * * @param componentFile any component file * @return The component sequence */ public static String getComponentSequence(String componentFile) { final ResourceReference ref = ResourceReference.of(componentFile); return IndexComponentFileReference.of(ref.getName()).getSequence(); }
protected void validateFiles(HashSet<String> groundTruth, ArrayList<IndexComponentFileReference> validFiles, FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, IBufferCache bufferCache) throws HyracksDataException { ArrayList<IndexComponentFileReference> tmpAllInvListsFiles = new ArrayList<>(); cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles, bufferCache); for (IndexComponentFileReference cmpFileName : tmpAllInvListsFiles) { if (groundTruth.contains(cmpFileName.getSequence())) { validFiles.add(cmpFileName); } else { delete(bufferCache, cmpFileName.getFullPath()); } } }
private long getOnDiskLastUsedComponentSequence(FilenameFilter filenameFilter) throws HyracksDataException { long maxComponentSeq = -1; final String[] files = listDirFiles(baseDir, filenameFilter); for (String fileName : files) { maxComponentSeq = Math.max(maxComponentSeq, IndexComponentFileReference.of(fileName).getSequenceEnd()); } return maxComponentSeq; } }
public static String getMergeSequence(String firstComponentName, String lastComponentName) { long mergeSequenceStart = IndexComponentFileReference.of(firstComponentName).getSequenceStart(); long mergeSequenceEnd = IndexComponentFileReference.of(lastComponentName).getSequenceEnd(); return mergeSequenceStart + DELIMITER + mergeSequenceEnd; } }
public boolean isWithin(IndexComponentFileReference other) { return sequenceStart >= other.getSequenceStart() && sequenceEnd <= other.getSequenceEnd(); }
validFiles.add(new LSMComponentFileReferences(allFiles.get(0).getFileRef(), null, null)); return validFiles; for (int i = 1; i < allFiles.size(); i++) { IndexComponentFileReference current = allFiles.get(i); if (current.isMoreRecentThan(last)) { } else if (current.isWithin(last)) { delete(treeFactory.getBufferCache(), current.getFullPath()); } else { validFiles.add(new LSMComponentFileReferences(cmpFileName.getFileRef(), null, null));
@Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) { final String baseName = IndexComponentFileReference.getMergeSequence(firstFileName, lastFileName); return new LSMComponentFileReferences(baseDir.getChild(baseName), null, null); }
private void deleteIndexInvalidComponents(File index) throws IOException, ParseException { final File[] indexComponentFiles = index.listFiles(COMPONENT_FILES_FILTER); if (indexComponentFiles == null) { throw new IOException(index + " doesn't exist or an IO error occurred"); } final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence(); for (File componentFile : indexComponentFiles) { // delete any file with start sequence > valid component sequence final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart(); if (fileStart > validComponentSequence) { LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath()); Files.delete(componentFile.toPath()); } } }
public static IndexComponentFileReference of(FileReference fileRef) { final IndexComponentFileReference ref = of(fileRef.getFile().getName()); ref.fileRef = fileRef; ref.fullPath = fileRef.getFile().getAbsolutePath(); return ref; }
public boolean isMoreRecentThan(IndexComponentFileReference other) { return sequenceStart > other.getSequenceEnd(); }
HashSet<String> deletedKeysBTreeFilesSet = new HashSet<>(); for (IndexComponentFileReference cmpFileName : allDeletedKeysBTreeFiles) { deletedKeysBTreeFilesSet.add(cmpFileName.getSequence()); validFiles.add(new LSMComponentFileReferences(allDictBTreeFiles.get(0).getFileRef(), allDeletedKeysBTreeFiles.get(0).getFileRef(), allBloomFilterFiles.get(0).getFileRef())); return validFiles; if (currentDeletedKeysBTree.isMoreRecentThan(lastDeletedKeysBTree) && currentDictBTree.isMoreRecentThan(lastDictBTree) && currentBloomFilter.isMoreRecentThan(lastBloomFilter)) { validComparableDictBTreeFiles.add(currentDictBTree); validComparableDeletedKeysBTreeFiles.add(currentDeletedKeysBTree); lastDeletedKeysBTree = currentDeletedKeysBTree; lastBloomFilter = currentBloomFilter; } else if (currentDeletedKeysBTree.isWithin(lastDeletedKeysBTree) && currentDictBTree.isWithin(lastDictBTree) && currentBloomFilter.isWithin(lastBloomFilter)) { delete(treeFactory.getBufferCache(), currentDeletedKeysBTree.getFullPath()); delete(treeFactory.getBufferCache(), currentDictBTree.getFullPath()); delete(treeFactory.getBufferCache(), currentBloomFilter.getFullPath()); } else { IndexComponentFileReference cmpDeletedKeysBTreeFile = deletedKeysBTreeIter.next(); IndexComponentFileReference cmpBloomFilterFileName = bloomFilterFileIter.next(); validFiles.add(new LSMComponentFileReferences(cmpDictBTreeFile.getFileRef(), cmpDeletedKeysBTreeFile.getFileRef(), cmpBloomFilterFileName.getFileRef()));
@Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) { final String baseName = IndexComponentFileReference.getMergeSequence(firstFileName, lastFileName); return new LSMComponentFileReferences(getFileReference(baseName + DELIMITER + BTREE_SUFFIX), null, hasBloomFilter ? getFileReference(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); }
@Override public int compare(IndexComponentFileReference a, IndexComponentFileReference b) { int startCmp = -Long.compare(a.getSequenceStart(), b.getSequenceStart()); if (startCmp != 0) { return startCmp; } return -Long.compare(a.getSequenceEnd(), b.getSequenceEnd()); } }
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<IndexComponentFileReference> allFiles, IBufferCache bufferCache) throws HyracksDataException { String[] files = listDirFiles(baseDir, filter); for (String fileName : files) { FileReference fileRef = getFileReference(fileName); if (treeFactory == null) { allFiles.add(IndexComponentFileReference.of(fileRef)); continue; } TreeIndexState idxState = isValidTreeIndex(treeFactory.createIndexInstance(fileRef)); if (idxState == TreeIndexState.VALID) { allFiles.add(IndexComponentFileReference.of(fileRef)); } else if (idxState == TreeIndexState.INVALID) { bufferCache.deleteFile(fileRef); } } }
HashSet<String> btreeFilesSet = new HashSet<>(); for (IndexComponentFileReference cmpFileName : allBTreeFiles) { int index = cmpFileName.getFileName().lastIndexOf(DELIMITER); btreeFilesSet.add(cmpFileName.getFileName().substring(0, index)); validFiles.add(new LSMComponentFileReferences(allBTreeFiles.get(0).getFileRef(), null, hasBloomFilter ? allBloomFilterFiles.get(0).getFileRef() : null)); return validFiles; if (currentBTree.isMoreRecentThan(lastBTree) && (!hasBloomFilter || currentBloomFilter.isMoreRecentThan(lastBloomFilter))) { validComparableBTreeFiles.add(currentBTree); lastBTree = currentBTree; lastBloomFilter = currentBloomFilter; } else if (currentBTree.isWithin(lastBTree) && (!hasBloomFilter || currentBloomFilter.isWithin(lastBloomFilter))) { delete(btreeFactory.getBufferCache(), currentBTree.getFullPath()); if (hasBloomFilter) { delete(btreeFactory.getBufferCache(), currentBloomFilter.getFullPath()); cmpBloomFilterFileName = bloomFilterFileIter.next(); validFiles.add(new LSMComponentFileReferences(cmpBTreeFileName.getFileRef(), null, hasBloomFilter ? cmpBloomFilterFileName.getFileRef() : null));
public Optional<Long> getLatestDiskComponentSequence() { if (diskComponents.isEmpty()) { return Optional.empty(); } final ILSMDiskComponent latestDiskComponent = diskComponents.get(0); final Set<String> diskComponentPhysicalFiles = latestDiskComponent.getLSMComponentPhysicalFiles(); final String fileName = diskComponentPhysicalFiles.stream().findAny() .orElseThrow(() -> new IllegalStateException("Disk component without any physical files")); return Optional .of(IndexComponentFileReference.of(Paths.get(fileName).getFileName().toString()).getSequenceEnd()); }
@Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) { final String baseName = IndexComponentFileReference.getMergeSequence(firstFileName, lastFileName); return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + RTREE_SUFFIX), baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); }
HashSet<String> btreeFilesSet = new HashSet<>(); for (IndexComponentFileReference cmpFileName : allBTreeFiles) { btreeFilesSet.add(cmpFileName.getSequence()); validFiles.add(new LSMComponentFileReferences(allRTreeFiles.get(0).getFileRef(), allBTreeFiles.get(0).getFileRef(), allBloomFilterFiles.get(0).getFileRef())); return validFiles; if (currentRTree.isMoreRecentThan(lastRTree) && currentBTree.isMoreRecentThan(lastBTree) && currentBloomFilter.isMoreRecentThan(lastBloomFilter)) { validComparableRTreeFiles.add(currentRTree); validComparableBTreeFiles.add(currentBTree); lastBTree = currentBTree; lastBloomFilter = currentBloomFilter; } else if (currentRTree.isWithin(lastRTree) && currentBTree.isWithin(lastBTree) && currentBloomFilter.isWithin(lastBloomFilter)) { delete(treeFactory.getBufferCache(), currentRTree.getFullPath()); delete(treeFactory.getBufferCache(), currentBTree.getFullPath()); delete(treeFactory.getBufferCache(), currentBloomFilter.getFullPath()); } else { IndexComponentFileReference cmpBTreeFileName = btreeFileIter.next(); IndexComponentFileReference cmpBloomFilterFileName = bloomFilterFileIter.next(); validFiles.add(new LSMComponentFileReferences(cmpRTreeFileName.getFileRef(), cmpBTreeFileName.getFileRef(), cmpBloomFilterFileName.getFileRef()));
private void updateBulkLoadedLastComponentSequence(INcApplicationContext appCtx) throws HyracksDataException { final ResourceReference indexRef = ResourceReference.of(file); final IIndexCheckpointManagerProvider checkpointManagerProvider = appCtx.getIndexCheckpointManagerProvider(); final IIndexCheckpointManager indexCheckpointManager = checkpointManagerProvider.get(indexRef); final long componentSequence = IndexComponentFileReference.of(indexRef.getName()).getSequenceEnd(); indexCheckpointManager.advanceValidComponentSequence(componentSequence); }