private Writer( File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits, int maxSegment ) throws IOException { this.path = file; this.serializer = serializer; this.codec = codec; this.blockSize = blockSize; this.recordIndexBits = recordIndexBits; this.padBits = padBits; segmentShift = 64-fileIndexBits; this.tmpPath = new File(file, "tmp"); this.rollFrequency = rollFrequency; tmpPath.mkdirs(); if (maxSegment < 0) { currentSegmentNum = getMaxSegmentNum(path); if (currentSegmentNum == -1 || verifySegmentIntegrity(path, currentSegmentNum)) currentSegmentNum++; } else { currentSegmentNum = maxSegment+1; } log.info("current segment num: "+currentSegmentNum); currentWriter = createWriter(currentSegmentNum); lastRollTime = System.currentTimeMillis(); }
/** * Creates a new segment file and flushes buffered writes to disk. * Provided metadata is written to record_log_directory_root/metadata.json * * @param metadata a mutable map of metadata to be written, may be empty. * @throws IOException if an I/O error occurs */ public synchronized void flushWriter(@Nonnull Map<String, String> metadata) throws IOException { writer.roll(); maxSegment = (int)(lastPosition >>> (64-RecordLogDirectory.DEFAULT_FILE_INDEX_BITS)); metadata.put(LAST_POSITION_KEY, String.valueOf(lastPosition)); metadata.put(MAX_SEGMENT_KEY, String.valueOf(maxSegment)); writeStringToFile(metadataPath, mapper.writeValueAsString(metadata)); writeStringToFile(lastPositionPath, String.valueOf(lastPosition)); writeStringToFile(maxSegmentPath, String.valueOf(maxSegment)); }
private Writer( File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits, int maxSegment ) throws IOException { this.path = file; this.serializer = serializer; this.codec = codec; this.blockSize = blockSize; this.recordIndexBits = recordIndexBits; this.padBits = padBits; segmentShift = 64-fileIndexBits; this.tmpPath = new File(file, "tmp"); this.rollFrequency = rollFrequency; tmpPath.mkdirs(); if (maxSegment < 0) { currentSegmentNum = getMaxSegmentNum(path); if (currentSegmentNum == -1 || verifySegmentIntegrity(path, currentSegmentNum)) currentSegmentNum++; } else { currentSegmentNum = maxSegment+1; } log.info("current segment num: "+currentSegmentNum); currentWriter = createWriter(currentSegmentNum); lastRollTime = System.currentTimeMillis(); }
/** * Creates a new segment file and flushes buffered writes to disk. * Provided metadata is written to record_log_directory_root/metadata.json * * @param metadata a mutable map of metadata to be written, may be empty. * @throws IOException if an I/O error occurs */ public synchronized void flushWriter(@Nonnull Map<String, String> metadata) throws IOException { writer.roll(); maxSegment = (int)(lastPosition >>> (64-RecordLogDirectory.DEFAULT_FILE_INDEX_BITS)); metadata.put(LAST_POSITION_KEY, String.valueOf(lastPosition)); metadata.put(MAX_SEGMENT_KEY, String.valueOf(maxSegment)); writeStringToFile(metadataPath, mapper.writeValueAsString(metadata)); writeStringToFile(lastPositionPath, String.valueOf(lastPosition)); writeStringToFile(maxSegmentPath, String.valueOf(maxSegment)); }
public static <E> Writer<E> create(File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency) throws IOException { return new Writer(file, serializer, codec, rollFrequency, DEFAULT_BLOCK_SIZE, DEFAULT_FILE_INDEX_BITS, DEFAULT_RECORD_INDEX_BITS, DEFAULT_PAD_BITS, -1); }
public static <E> Writer<E> create(File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency) throws IOException { return new Writer(file, serializer, codec, rollFrequency, DEFAULT_BLOCK_SIZE, DEFAULT_FILE_INDEX_BITS, DEFAULT_RECORD_INDEX_BITS, DEFAULT_PAD_BITS, -1); }
public static <E> Writer<E> create( File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits, int maxSegment ) throws IOException { return new Writer(file, serializer, codec, rollFrequency, blockSize, fileIndexBits, recordIndexBits, padBits, maxSegment); }
public static <E> Writer<E> create( File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits ) throws IOException { return new Writer(file, serializer, codec, rollFrequency, blockSize, fileIndexBits, recordIndexBits, padBits, -1); }
public void roll() throws IOException { currentWriter.sync(); currentWriter.close(); final File segmentFile = getSegmentPath(path, currentSegmentNum, true); currentWriterPath.renameTo(segmentFile); currentWriter = createWriter(++currentSegmentNum); lastRollTime = System.currentTimeMillis(); }
@Override public void close() throws IOException { try { roll(); } finally { currentWriter.close(); } }
@Override public long append(final E entry) throws IOException { if (System.currentTimeMillis()-lastRollTime > rollFrequency) { roll(); } final long writerAddress = currentWriter.append(entry); if (writerAddress >= 1L<< segmentShift) throw new IOException("current writer has exceeded maximum size"); return (((long)currentSegmentNum)<< segmentShift)+writerAddress; }
public static <E> Writer<E> create( File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits ) throws IOException { return new Writer(file, serializer, codec, rollFrequency, blockSize, fileIndexBits, recordIndexBits, padBits, -1); }
@Override public void close() throws IOException { try { roll(); } finally { currentWriter.close(); } }
public static <E> Writer<E> create( File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits, int maxSegment ) throws IOException { return new Writer(file, serializer, codec, rollFrequency, blockSize, fileIndexBits, recordIndexBits, padBits, maxSegment); }
@Override public long append(final E entry) throws IOException { if (System.currentTimeMillis()-lastRollTime > rollFrequency) { roll(); } final long writerAddress = currentWriter.append(entry); if (writerAddress >= 1L<< segmentShift) throw new IOException("current writer has exceeded maximum size"); return (((long)currentSegmentNum)<< segmentShift)+writerAddress; }
/** * Writes an entry to the record file. This write is not guaranteed to be persisted * unless {@link #flushWriter(java.util.Map)} is called. * * @param op operation to write * @return address operation was written to * @throws IOException if an I/O error occurs */ protected long writeOperation(final T op) throws IOException { lastPosition = writer.append(op); return lastPosition; }
/** * Writes an entry to the record file. This write is not guaranteed to be persisted * unless {@link #flushWriter(java.util.Map)} is called. * * @param op operation to write * @return address operation was written to * @throws IOException if an I/O error occurs */ protected long writeOperation(final T op) throws IOException { lastPosition = writer.append(op); return lastPosition; }
public void roll() throws IOException { currentWriter.sync(); currentWriter.close(); final File segmentFile = getSegmentPath(path, currentSegmentNum, true); currentWriterPath.renameTo(segmentFile); currentWriter = createWriter(++currentSegmentNum); lastRollTime = System.currentTimeMillis(); }