public boolean load() { File dir = new File(this.storePath); File[] files = dir.listFiles(); if (files != null) { // ascending order Arrays.sort(files); for (File file : files) { if (file.length() != this.mappedFileSize) { log.warn(file + "\t" + file.length() + " length not matched message store config value, ignore it"); return true; } try { MappedFile mappedFile = new MappedFile(file.getPath(), mappedFileSize); mappedFile.setWrotePosition(this.mappedFileSize); mappedFile.setFlushedPosition(this.mappedFileSize); mappedFile.setCommittedPosition(this.mappedFileSize); this.mappedFiles.add(mappedFile); log.info("load " + file.getPath() + " OK"); } catch (IOException e) { log.error("load file " + file + " error", e); return false; } } } return true; }
if (mappedFile != null) { if (mappedFile.isFirstCreateInQueue() && cqOffset != 0 && mappedFile.getWrotePosition() == 0) { this.minLogicOffset = expectLogicOffset; this.mappedFileQueue.setFlushedWhere(expectLogicOffset); this.mappedFileQueue.setCommittedWhere(expectLogicOffset); this.fillPreBlank(mappedFile, expectLogicOffset); log.info("fill pre blank space " + mappedFile.getFileName() + " " + expectLogicOffset + " " + mappedFile.getWrotePosition()); long currentLogicOffset = mappedFile.getWrotePosition() + mappedFile.getFileFromOffset(); return mappedFile.appendMessage(this.byteBufferIndex.array());
public AppendMessageResult appendMessages(final MessageExtBatch messageExtBatch, final AppendMessageCallback cb) { return appendMessagesInner(messageExtBatch, cb); }
public boolean flush(final int flushLeastPages) { boolean result = true; MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); if (mappedFile != null) { long tmpTimeStamp = mappedFile.getStoreTimestamp(); int offset = mappedFile.flush(flushLeastPages); long where = mappedFile.getFileFromOffset() + offset; result = where == this.flushedWhere; this.flushedWhere = where; if (0 == flushLeastPages) { this.storeTimestamp = tmpTimeStamp; } } return result; }
public void truncateDirtyFiles(long offset) { List<MappedFile> willRemoveFiles = new ArrayList<MappedFile>(); for (MappedFile file : this.mappedFiles) { long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; if (fileTailOffset > offset) { if (offset >= file.getFileFromOffset()) { file.setWrotePosition((int) (offset % this.mappedFileSize)); file.setCommittedPosition((int) (offset % this.mappedFileSize)); file.setFlushedPosition((int) (offset % this.mappedFileSize)); } else { file.destroy(1000); willRemoveFiles.add(file); } } } this.deleteExpiredFile(willRemoveFiles); }
public boolean commit(final int commitLeastPages) { boolean result = true; MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); if (mappedFile != null) { int offset = mappedFile.commit(commitLeastPages); long where = mappedFile.getFileFromOffset() + offset; result = where == this.committedWhere; this.committedWhere = where; } return result; }
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile == null || mappedFile.isFull()) { mappedFile = this.mappedFileQueue.getLastMappedFile(0); continue; final int wrotePosition = mappedFile.getWrotePosition(); final int blankSize = this.mappedFileSize - wrotePosition - END_BLANK_DATA_LENGTH; fullFillToEnd(mappedFile, wrotePosition); log.info("No enough space(need:{}, has:{}) of file {}, so fill to end", size, blankSize, mappedFile.getFileName()); continue; if (mappedFile.appendMessage(cqExtUnit.write(this.tempContainer), 0, size)) { return decorate(wrotePosition + mappedFile.getFileFromOffset());
public int commit(final int commitLeastPages) { if (writeBuffer == null) { //no need to commit data to file channel, so just regard wrotePosition as committedPosition. return this.wrotePosition.get(); } if (this.isAbleToCommit(commitLeastPages)) { if (this.hold()) { commit0(commitLeastPages); this.release(); } else { log.warn("in commit, hold failed, commit offset = " + this.committedPosition.get()); } } // All dirty data has been committed to FileChannel. if (writeBuffer != null && this.transientStorePool != null && this.fileSize == this.committedPosition.get()) { this.transientStorePool.returnBuffer(writeBuffer); this.writeBuffer = null; } return this.committedPosition.get(); }
if (mappedFileLast != null && mappedFileLast.isFull()) { createOffset = mappedFileLast.getFileFromOffset() + this.mappedFileSize; } else { try { mappedFile = new MappedFile(nextFilePath, this.mappedFileSize); } catch (IOException e) { log.error("create mappedFile exception", e); mappedFile.setFirstCreateInQueue(true);
try { mappedFile = ServiceLoader.load(MappedFile.class).iterator().next(); mappedFile.init(req.getFilePath(), req.getFileSize(), messageStore.getTransientStorePool()); } catch (RuntimeException e) { log.warn("Use default implementation."); mappedFile = new MappedFile(req.getFilePath(), req.getFileSize(), messageStore.getTransientStorePool()); mappedFile = new MappedFile(req.getFilePath(), req.getFileSize()); if (mappedFile.getFileSize() >= this.messageStore.getMessageStoreConfig() .getMapedFileSizeCommitLog() && this.messageStore.getMessageStoreConfig().isWarmMapedFileEnable()) { mappedFile.warmMappedFile(this.messageStore.getMessageStoreConfig().getFlushDiskType(), this.messageStore.getMessageStoreConfig().getFlushLeastPagesWhenWarmMapedFile());
@Override public boolean cleanup(final long currentRef) { if (this.isAvailable()) { log.error("this file[REF:" + currentRef + "] " + this.fileName + " have not shutdown, stop unmapping."); return false; } if (this.isCleanupOver()) { log.error("this file[REF:" + currentRef + "] " + this.fileName + " have cleanup, do not do it again."); return true; } clean(this.mappedByteBuffer); TOTAL_MAPPED_VIRTUAL_MEMORY.addAndGet(this.fileSize * (-1)); TOTAL_MAPPED_FILES.decrementAndGet(); log.info("unmap file[REF:" + currentRef + "] " + this.fileName + " OK"); return true; }
public boolean retryDeleteFirstFile(final long intervalForcibly) { MappedFile mappedFile = this.getFirstMappedFile(); if (mappedFile != null) { if (!mappedFile.isAvailable()) { log.warn("the mappedFile was destroyed once, but still alive, " + mappedFile.getFileName()); boolean result = mappedFile.destroy(intervalForcibly); if (result) { log.info("the mappedFile re delete OK, " + mappedFile.getFileName()); List<MappedFile> tmpFiles = new ArrayList<MappedFile>(); tmpFiles.add(mappedFile); this.deleteExpiredFile(tmpFiles); } else { log.warn("the mappedFile re delete failed, " + mappedFile.getFileName()); } return result; } } return false; }
public IndexFile(final String fileName, final int hashSlotNum, final int indexNum, final long endPhyOffset, final long endTimestamp) throws IOException { int fileTotalSize = IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * hashSlotSize) + (indexNum * indexSize); this.mappedFile = new MappedFile(fileName, fileTotalSize); this.fileChannel = this.mappedFile.getFileChannel(); this.mappedByteBuffer = this.mappedFile.getMappedByteBuffer(); this.hashSlotNum = hashSlotNum; this.indexNum = indexNum; ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); this.indexHeader = new IndexHeader(byteBuffer); if (endPhyOffset > 0) { this.indexHeader.setBeginPhyOffset(endPhyOffset); this.indexHeader.setEndPhyOffset(endPhyOffset); } if (endTimestamp > 0) { this.indexHeader.setBeginTimestamp(endTimestamp); this.indexHeader.setEndTimestamp(endTimestamp); } }
/** * Delete files before {@code minAddress}. * * @param minAddress less than 0 */ public void truncateByMinAddress(final long minAddress) { if (!isExtAddr(minAddress)) { return; } log.info("Truncate consume queue ext by min {}.", minAddress); List<MappedFile> willRemoveFiles = new ArrayList<MappedFile>(); List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles(); final long realOffset = unDecorate(minAddress); for (MappedFile file : mappedFiles) { long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; if (fileTailOffset < realOffset) { log.info("Destroy consume queue ext by min: file={}, fileTailOffset={}, minOffset={}", file.getFileName(), fileTailOffset, realOffset); if (file.destroy(1000)) { willRemoveFiles.add(file); } } } this.mappedFileQueue.deleteExpiredFile(willRemoveFiles); }
boolean destroy; MappedFile mappedFile = (MappedFile) mfs[i]; SelectMappedBufferResult result = mappedFile.selectMappedBuffer(this.mappedFileSize - unitSize); if (result != null) { long maxOffsetInLogicQueue = result.getByteBuffer().getLong(); + maxOffsetInLogicQueue + ", delete it"); } else if (!mappedFile.isAvailable()) { // Handle hanged file. log.warn("Found a hanged consume queue file, attempting to delete it."); destroy = true; if (destroy && mappedFile.destroy(1000 * 60)) { files.add(mappedFile); deleteCount++;
public void deleteLastMappedFile() { MappedFile lastMappedFile = getLastMappedFile(); if (lastMappedFile != null) { lastMappedFile.destroy(1000); this.mappedFiles.remove(lastMappedFile); log.info("on recover, destroy a logic mapped file " + lastMappedFile.getFileName()); } }
public void backupFiles(long offset, String backupPath) { for (MappedFile file : this.mappedFiles) { long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; if (fileTailOffset > offset) { file.flush(0); try { Path source = Paths.get(file.getFileName()); Path target = Paths.get(StorePathConfigHelper.getBackupStoreFilePath(backupPath, new File(file.getFileName()).getName())); Files.copy(source, target); log.info("backup file:{} to {}", file.getFileName(), target); } catch (Exception ex) { log.error("backup file failed", ex); } } } }
/** * @throws IOException */ private void createTempFile() throws IOException { String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()); File file = new File(fileName); MappedFile.ensureDirOK(file.getParent()); boolean result = file.createNewFile(); log.info(fileName + (result ? " create OK" : " already exists")); }
for (int i = 0; i < mfsLength; i++) { MappedFile mappedFile = (MappedFile) mfs[i]; long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime; if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) { if (mappedFile.destroy(intervalForcibly)) { files.add(mappedFile); deleteCount++;
if (null == mappedFile || mappedFile.isFull()) { mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback); switch (result.getStatus()) { case PUT_OK: return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback); break; case MESSAGE_SIZE_EXCEEDED: