public static String getMergeSequence(String firstComponentName, String lastComponentName) { long mergeSequenceStart = IndexComponentFileReference.of(firstComponentName).getSequenceStart(); long mergeSequenceEnd = IndexComponentFileReference.of(lastComponentName).getSequenceEnd(); return mergeSequenceStart + DELIMITER + mergeSequenceEnd; } }
public Optional<Long> getLatestDiskComponentSequence() { if (diskComponents.isEmpty()) { return Optional.empty(); } final ILSMDiskComponent latestDiskComponent = diskComponents.get(0); final Set<String> diskComponentPhysicalFiles = latestDiskComponent.getLSMComponentPhysicalFiles(); final String fileName = diskComponentPhysicalFiles.stream().findAny() .orElseThrow(() -> new IllegalStateException("Disk component without any physical files")); return Optional .of(IndexComponentFileReference.of(Paths.get(fileName).getFileName().toString()).getSequenceEnd()); }
public static IndexComponentFileReference of(FileReference fileRef) { final IndexComponentFileReference ref = of(fileRef.getFile().getName()); ref.fileRef = fileRef; ref.fullPath = fileRef.getFile().getAbsolutePath(); return ref; }
private long getOnDiskLastUsedComponentSequence(FilenameFilter filenameFilter) throws HyracksDataException { long maxComponentSeq = -1; final String[] files = listDirFiles(baseDir, filenameFilter); for (String fileName : files) { maxComponentSeq = Math.max(maxComponentSeq, IndexComponentFileReference.of(fileName).getSequenceEnd()); } return maxComponentSeq; } }
private void deleteIndexInvalidComponents(File index) throws IOException, ParseException { final File[] indexComponentFiles = index.listFiles(COMPONENT_FILES_FILTER); if (indexComponentFiles == null) { throw new IOException(index + " doesn't exist or an IO error occurred"); } final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence(); for (File componentFile : indexComponentFiles) { // delete any file with start sequence > valid component sequence final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart(); if (fileStart > validComponentSequence) { LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath()); Files.delete(componentFile.toPath()); } } }
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter, TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<IndexComponentFileReference> allFiles, IBufferCache bufferCache) throws HyracksDataException { String[] files = listDirFiles(baseDir, filter); for (String fileName : files) { FileReference fileRef = getFileReference(fileName); if (treeFactory == null) { allFiles.add(IndexComponentFileReference.of(fileRef)); continue; } TreeIndexState idxState = isValidTreeIndex(treeFactory.createIndexInstance(fileRef)); if (idxState == TreeIndexState.VALID) { allFiles.add(IndexComponentFileReference.of(fileRef)); } else if (idxState == TreeIndexState.INVALID) { bufferCache.deleteFile(fileRef); } } }
/** * Gets a component sequence based on its unique timestamp. * e.g. a component file 1_3_b * will return a component sequence 1_3 * * @param componentFile any component file * @return The component sequence */ public static String getComponentSequence(String componentFile) { final ResourceReference ref = ResourceReference.of(componentFile); return IndexComponentFileReference.of(ref.getName()).getSequence(); }
private void ensureComponentLsnFlushed(INcApplicationContext appCtx) throws HyracksDataException, InterruptedException { final ResourceReference indexRef = ResourceReference.of(file); final IIndexCheckpointManagerProvider checkpointManagerProvider = appCtx.getIndexCheckpointManagerProvider(); final IIndexCheckpointManager indexCheckpointManager = checkpointManagerProvider.get(indexRef); long replicationTimeOut = TimeUnit.SECONDS.toMillis(appCtx.getReplicationProperties().getReplicationTimeOut()); synchronized (indexCheckpointManager) { // wait until the lsn mapping is flushed to disk while (!indexCheckpointManager.isFlushed(masterLsn)) { if (replicationTimeOut <= 0) { throw new ReplicationException(new TimeoutException("Couldn't receive flush lsn from master")); } final long startTime = System.nanoTime(); indexCheckpointManager.wait(replicationTimeOut); replicationTimeOut -= TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); } final long componentSequence = IndexComponentFileReference.of(indexRef.getName()).getSequenceEnd(); indexCheckpointManager.replicated(componentSequence, masterLsn, lastComponentId); } }
private void addComponentToCheckpoint(ILSMIOOperation operation) throws HyracksDataException { // will always update the checkpoint file even if no new component was created FileReference target = operation.getTarget(); Map<String, Object> map = operation.getParameters(); final Long lsn = operation.getIOOpertionType() == LSMIOOperationType.FLUSH ? (Long) map.get(KEY_FLUSH_LOG_LSN) : 0L; final LSMComponentId id = (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID); final ResourceReference ref = ResourceReference.of(target.getAbsolutePath()); final long componentSequence = IndexComponentFileReference.of(ref.getName()).getSequenceEnd(); indexCheckpointManagerProvider.get(ref).flushed(componentSequence, lsn, id.getMaxId()); }
private void updateBulkLoadedLastComponentSequence(INcApplicationContext appCtx) throws HyracksDataException { final ResourceReference indexRef = ResourceReference.of(file); final IIndexCheckpointManagerProvider checkpointManagerProvider = appCtx.getIndexCheckpointManagerProvider(); final IIndexCheckpointManager indexCheckpointManager = checkpointManagerProvider.get(indexRef); final long componentSequence = IndexComponentFileReference.of(indexRef.getName()).getSequenceEnd(); indexCheckpointManager.advanceValidComponentSequence(componentSequence); }
@Override public void perform(INcApplicationContext appCtx, IReplicationWorker worker) throws HyracksDataException { final IIndexCheckpointManagerProvider indexCheckpointManagerProvider = appCtx.getIndexCheckpointManagerProvider(); PersistentLocalResourceRepository resRepo = (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository(); final IIOManager ioManager = appCtx.getIoManager(); final Collection<LocalResource> partitionResources = resRepo.getPartitionResources(partition).values(); final long currentLSN = appCtx.getTransactionSubsystem().getLogManager().getAppendLSN(); for (LocalResource ls : partitionResources) { DatasetResourceReference ref = DatasetResourceReference.of(ls); final IIndexCheckpointManager indexCheckpointManager = indexCheckpointManagerProvider.get(ref); indexCheckpointManager.delete(); // Get most recent sequence of existing files to avoid deletion Path indexPath = StoragePathUtil.getIndexPath(ioManager, ref); String[] files = indexPath.toFile().list(AbstractLSMIndexFileManager.COMPONENT_FILES_FILTER); if (files == null) { throw HyracksDataException .create(new IOException(indexPath + " is not a directory or an IO Error occurred")); } long maxComponentSequence = Long.MIN_VALUE; for (String file : files) { maxComponentSequence = Math.max(maxComponentSequence, IndexComponentFileReference.of(file).getSequenceEnd()); } indexCheckpointManager.init(maxComponentSequence, currentLSN, maxComponentId); } ReplicationProtocol.sendAck(worker.getChannel(), worker.getReusableBuffer()); }