/** * Constructor. * * @param topic topic * @param queueId id of queue * @param storePath root dir of files to store. * @param mappedFileSize file size * @param bitMapLength bit map length. */ public ConsumeQueueExt(final String topic, final int queueId, final String storePath, final int mappedFileSize, final int bitMapLength) { this.storePath = storePath; this.mappedFileSize = mappedFileSize; this.topic = topic; this.queueId = queueId; String queueDir = this.storePath + File.separator + topic + File.separator + queueId; this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null); if (bitMapLength > 0) { this.tempContainer = ByteBuffer.allocate( bitMapLength / Byte.SIZE ); } }
/** * Check whether the step size in mapped file queue is correct. */ public void checkSelf() { this.mappedFileQueue.checkSelf(); }
public long flush() { this.mappedFileQueue.commit(0); this.mappedFileQueue.flush(0); return this.mappedFileQueue.getFlushedWhere(); }
final long intervalForcibly, final boolean cleanImmediately) { Object[] mfs = this.copyMappedFiles(0); deleteExpiredFile(files);
public boolean retryDeleteFirstFile(final long intervalForcibly) { MappedFile mappedFile = this.getFirstMappedFile(); if (mappedFile != null) { if (!mappedFile.isAvailable()) { log.warn("the mappedFile was destroyed once, but still alive, " + mappedFile.getFileName()); boolean result = mappedFile.destroy(intervalForcibly); if (result) { log.info("the mappedFile re delete OK, " + mappedFile.getFileName()); List<MappedFile> tmpFiles = new ArrayList<MappedFile>(); tmpFiles.add(mappedFile); this.deleteExpiredFile(tmpFiles); } else { log.warn("the mappedFile re delete failed, " + mappedFile.getFileName()); } return result; } } return false; }
public MappedFile findMappedFileByOffset(final long offset) { return findMappedFileByOffset(offset, false); }
/** * Delete files before {@code minAddress}. * * @param minAddress less than 0 */ public void truncateByMinAddress(final long minAddress) { if (!isExtAddr(minAddress)) { return; } log.info("Truncate consume queue ext by min {}.", minAddress); List<MappedFile> willRemoveFiles = new ArrayList<MappedFile>(); List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles(); final long realOffset = unDecorate(minAddress); for (MappedFile file : mappedFiles) { long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; if (fileTailOffset < realOffset) { log.info("Destroy consume queue ext by min: file={}, fileTailOffset={}, minOffset={}", file.getFileName(), fileTailOffset, realOffset); if (file.destroy(1000)) { willRemoveFiles.add(file); } } } this.mappedFileQueue.deleteExpiredFile(willRemoveFiles); }
long maxExtAddr = 1; while (true) { MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile != null) { ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); this.mappedFileQueue.deleteLastMappedFile(); break; } else {
/** * delete files and directory. */ public void destroy() { this.mappedFileQueue.destroy(); }
public MappedFile getMappedFileByTime(final long timestamp) { Object[] mfs = this.copyMappedFiles(0); if (null == mfs) return null; for (int i = 0; i < mfs.length; i++) { MappedFile mappedFile = (MappedFile) mfs[i]; if (mappedFile.getLastModifiedTimestamp() >= timestamp) { return mappedFile; } } return (MappedFile) mfs[mfs.length - 1]; }
public int deleteExpiredFile( final long expiredTime, final int deleteFilesInterval, final long intervalForcibly, final boolean cleanImmediately ) { return this.mappedFileQueue.deleteExpiredFileByTime(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); }
public int deleteExpiredFile(long offset) { int cnt = this.mappedFileQueue.deleteExpiredFileByOffset(offset, CQ_STORE_UNIT_SIZE); this.correctMinOffset(offset); return cnt; }
public void truncateDirtyFiles(long offset) { List<MappedFile> willRemoveFiles = new ArrayList<MappedFile>(); for (MappedFile file : this.mappedFiles) { long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; if (fileTailOffset > offset) { if (offset >= file.getFileFromOffset()) { file.setWrotePosition((int) (offset % this.mappedFileSize)); file.setCommittedPosition((int) (offset % this.mappedFileSize)); file.setFlushedPosition((int) (offset % this.mappedFileSize)); } else { file.destroy(1000); willRemoveFiles.add(file); } } } this.deleteExpiredFile(willRemoveFiles); }
boolean result = CommitLog.this.mappedFileQueue.commit(commitDataLeastPages); long end = System.currentTimeMillis(); if (!result) { result = CommitLog.this.mappedFileQueue.commit(0); CommitLog.log.info(this.getServiceName() + " service shutdown, retry " + (i + 1) + " times " + (result ? "OK" : "Not OK"));
public int deleteExpiredFileByOffset(long offset, int unitSize) { Object[] mfs = this.copyMappedFiles(0); deleteExpiredFile(files);
public boolean commit(final int commitLeastPages) { boolean result = true; MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); if (mappedFile != null) { int offset = mappedFile.commit(commitLeastPages); long where = mappedFile.getFileFromOffset() + offset; result = where == this.committedWhere; this.committedWhere = where; } return result; }
public boolean retryDeleteFirstFile(final long intervalForcibly) { MappedFile mappedFile = this.getFirstMappedFile(); if (mappedFile != null) { if (!mappedFile.isAvailable()) { log.warn("the mappedFile was destroyed once, but still alive, " + mappedFile.getFileName()); boolean result = mappedFile.destroy(intervalForcibly); if (result) { log.info("the mappedFile re delete OK, " + mappedFile.getFileName()); List<MappedFile> tmpFiles = new ArrayList<MappedFile>(); tmpFiles.add(mappedFile); this.deleteExpiredFile(tmpFiles); } else { log.warn("the mappedFile re delete failed, " + mappedFile.getFileName()); } return result; } } return false; }
/** * Delete files before {@code minAddress}. * * @param minAddress less than 0 */ public void truncateByMinAddress(final long minAddress) { if (!isExtAddr(minAddress)) { return; } log.info("Truncate consume queue ext by min {}.", minAddress); List<MappedFile> willRemoveFiles = new ArrayList<MappedFile>(); List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles(); final long realOffset = unDecorate(minAddress); for (MappedFile file : mappedFiles) { long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; if (fileTailOffset < realOffset) { log.info("Destroy consume queue ext by min: file={}, fileTailOffset={}, minOffset={}", file.getFileName(), fileTailOffset, realOffset); if (file.destroy(1000)) { willRemoveFiles.add(file); } } } this.mappedFileQueue.deleteExpiredFile(willRemoveFiles); }
long maxExtAddr = 1; while (true) { MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile != null) { ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); this.mappedFileQueue.deleteLastMappedFile(); break; } else {
public void destroy() { this.mappedFileQueue.destroy(); }