/** * read <i>length</i> bytes from <i>position</i> of a file channel. * * @param channel channel to read from * @param channelPosition position to read from * @param dest destination byte array to put data in * @param destOffset offset in dest to read into * @param length number of bytes to read * @return total bytes read or -1 if an attempt was made to read past EOF. The method always tries to read all the bytes * that will fit in the destination byte buffer. */ public static int readFromFileChannel(FileChannel channel, long channelPosition, byte[] dest, int destOffset, int length) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(dest, destOffset, length); return readFromFileChannel(channel, channelPosition, buffer); }
/** * read <i>length</i> bytes from <i>position</i> of a file channel */ public static byte[] readFromFileChannel(FileChannel channel, long position, int length) throws IOException { byte[] res = new byte[length]; readFromFileChannelWithEofException(channel, position, res, 0, length); return res; }
return readSingleChunk(channel, channelPosition, dest); } else { int bytesRead = 0; while (dest.hasRemaining()) { tmpBuffer.limit(Math.min(dest.limit(), tmpBuffer.position() + READ_CHUNK_SIZE)); int read = readSingleChunk(channel, channelPosition, tmpBuffer); if (read < 0) { return read;
/** * Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel} * * @param source byte array to copy from * @param channel target WritableByteChannel */ public static void writeToChannel(byte[] source, WritableByteChannel channel) throws IOException { writeToChannel(source, 0, source.length, channel); }
public static void write(ChannelFactory factory, Path checkpointFile, Checkpoint checkpoint, OpenOption... options) throws IOException { final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(V3_FILE_SIZE) { @Override public synchronized byte[] toByteArray() { // don't clone return buf; } }; final String resourceDesc = "checkpoint(path=\"" + checkpointFile + "\", gen=" + checkpoint + ")"; try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, V3_FILE_SIZE)) { CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, CURRENT_VERSION); checkpoint.write(indexOutput); CodecUtil.writeFooter(indexOutput); assert indexOutput.getFilePointer() == V3_FILE_SIZE : "get you numbers straight; bytes written: " + indexOutput.getFilePointer() + ", buffer size: " + V3_FILE_SIZE; assert indexOutput.getFilePointer() < 512 : "checkpoint files have to be smaller than 512 bytes for atomic writes; size: " + indexOutput.getFilePointer(); } // now go and write to the channel, in one go. try (FileChannel channel = factory.open(checkpointFile, options)) { Channels.writeToChannel(byteOutputStream.toByteArray(), channel); // no need to force metadata, file size stays the same and we did the full fsync // when we first created the file, so the directory entry doesn't change as well channel.force(false); } }
/** * reads an operation at the given position into the given buffer. */ protected void readBytes(ByteBuffer buffer, long position) throws IOException { if (position >= length) { throw new EOFException("read requested past EOF. pos [" + position + "] end: [" + length + "]"); } if (position < getFirstOperationOffset()) { throw new IOException("read requested before position of first ops. pos [" + position + "] first op on: [" + getFirstOperationOffset() + "]"); } Channels.readFromFileChannelWithEofException(channel, position, buffer); }
/** * read <i>length</i> bytes from <i>position</i> of a file channel. An EOFException will be thrown if you * attempt to read beyond the end of file. * * @param channel channel to read from * @param channelPosition position to read from * @param dest destination byte array to put data in * @param destOffset offset in dest to read into * @param length number of bytes to read */ public static void readFromFileChannelWithEofException(FileChannel channel, long channelPosition, byte[] dest, int destOffset, int length) throws IOException { int read = readFromFileChannel(channel, channelPosition, dest, destOffset, length); if (read < 0) { throw new EOFException("read past EOF. pos [" + channelPosition + "] length: [" + length + "] end: [" + channel.size() + "]"); } }
/** * Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel} * * @param source byte array to copy from * @param channel target WritableByteChannel */ public static void writeToChannel(byte[] source, WritableByteChannel channel) throws IOException { writeToChannel(source, 0, source.length, channel); }
return readSingleChunk(channel, channelPosition, dest); } else { int bytesRead = 0; while (dest.hasRemaining()) { tmpBuffer.limit(Math.min(dest.limit(), tmpBuffer.position() + READ_CHUNK_SIZE)); int read = readSingleChunk(channel, channelPosition, tmpBuffer); if (read < 0) { return read;
@Override protected void readBytes(ByteBuffer targetBuffer, long position) throws IOException { try { if (position + targetBuffer.remaining() > getWrittenOffset()) { synchronized (this) { // we only flush here if it's really really needed - try to minimize the impact of the read operation // in some cases ie. a tragic event we might still be able to read the relevant value // which is not really important in production but some test can make most strict assumptions // if we don't fail in this call unless absolutely necessary. if (position + targetBuffer.remaining() > getWrittenOffset()) { outputStream.flush(); } } } } catch (final Exception ex) { closeWithTragicEvent(ex); throw ex; } // we don't have to have a lock here because we only write ahead to the file, so all writes has been complete // for the requested location. Channels.readFromFileChannelWithEofException(channel, position, targetBuffer); }
/** * read from a file channel into a byte buffer, starting at a certain position. An EOFException will be thrown if you * attempt to read beyond the end of file. * * @param channel channel to read from * @param channelPosition position to read from * @param dest destination {@link java.nio.ByteBuffer} to put data in */ public static void readFromFileChannelWithEofException(FileChannel channel, long channelPosition, ByteBuffer dest) throws IOException { int read = readFromFileChannel(channel, channelPosition, dest); if (read < 0) { throw new EOFException("read past EOF. pos [" + channelPosition + "] length: [" + dest.limit() + "] end: [" + channel.size() + "]"); } }
/** * Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel} * * @param source byte array to copy from * @param channel target WritableByteChannel */ public static void writeToChannel(byte[] source, WritableByteChannel channel) throws IOException { writeToChannel(source, 0, source.length, channel); }
return readSingleChunk(channel, channelPosition, dest); } else { int bytesRead = 0; while (dest.hasRemaining()) { tmpBuffer.limit(Math.min(dest.limit(), tmpBuffer.position() + READ_CHUNK_SIZE)); int read = readSingleChunk(channel, channelPosition, tmpBuffer); if (read < 0) { return read;
/** * reads an operation at the given position into the given buffer. */ protected void readBytes(ByteBuffer buffer, long position) throws IOException { if (position >= length) { throw new EOFException("read requested past EOF. pos [" + position + "] end: [" + length + "], generation: [" + getGeneration() + "], path: [" + path + "]"); } if (position < getFirstOperationOffset()) { throw new IOException("read requested before position of first ops. pos [" + position + "] first op on: [" + getFirstOperationOffset() + "], generation: [" + getGeneration() + "], path: [" + path + "]"); } Channels.readFromFileChannelWithEofException(channel, position, buffer); }
private static void tryReportOldVersionError(final Path path, final FileChannel channel) throws IOException { // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the header, in binary this looks like: // binary: 0011 1111 1101 0111 0110 1100 0001 0111 // hex : 3 f d 7 6 c 1 7 // // With version 0 of the translog, the first byte is the Operation.Type, which will always be between 0-4, // so we know if we grab the first byte, it can be: // 0x3f => Lucene's magic number, so we can assume it's version 1 or later // 0x00 => version 0 of the translog final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0]; if (b1 == 0x3f) { // LUCENE_CODEC_HEADER_BYTE throw new TranslogCorruptedException( path.toString(), "translog looks like version 1 or later, but has corrupted header" ); } else if (b1 == 0x00) { // UNVERSIONED_TRANSLOG_HEADER_BYTE throw new IllegalStateException("pre-1.4 translog found [" + path + "]"); } }
/** * Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel} * * @param source byte array to copy from * @param channel target WritableByteChannel */ public static void writeToChannel(byte[] source, WritableByteChannel channel) throws IOException { writeToChannel(source, 0, source.length, channel); }
return readSingleChunk(channel, channelPosition, dest); } else { int bytesRead = 0; while (dest.hasRemaining()) { tmpBuffer.limit(Math.min(dest.limit(), tmpBuffer.position() + READ_CHUNK_SIZE)); int read = readSingleChunk(channel, channelPosition, tmpBuffer); if (read < 0) { return read;
/** * read <i>length</i> bytes from <i>position</i> of a file channel */ public static byte[] readFromFileChannel(FileChannel channel, long position, int length) throws IOException { byte[] res = new byte[length]; readFromFileChannelWithEofException(channel, position, res, 0, length); return res; }
/** * read <i>length</i> bytes from <i>position</i> of a file channel. * * @param channel channel to read from * @param channelPosition position to read from * @param dest destination byte array to put data in * @param destOffset offset in dest to read into * @param length number of bytes to read * @return total bytes read or -1 if an attempt was made to read past EOF. The method always tries to read all the bytes * that will fit in the destination byte buffer. */ public static int readFromFileChannel(FileChannel channel, long channelPosition, byte[] dest, int destOffset, int length) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(dest, destOffset, length); return readFromFileChannel(channel, channelPosition, buffer); }
/** * Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel} * * @param source byte array to copy from * @param channel target WritableByteChannel */ public static void writeToChannel(byte[] source, WritableByteChannel channel) throws IOException { writeToChannel(source, 0, source.length, channel); }