/** * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must * guarantee that the file may be written to before calling this method. * * @param input iterator containing records * @param serializer serializer for the input records * @return size of the written buffer * @throws IOException */ protected <IN> int sendBuffer(SingleElementPushBackIterator<IN> input, Serializer<IN> serializer) throws IOException { fileBuffer.clear(); while (input.hasNext()) { IN value = input.next(); ByteBuffer bb = serializer.serialize(value); if (bb.remaining() > mappedFileSizeBytes) { throw new RuntimeException("Serialized object does not fit into a single buffer."); } if (bb.remaining() <= fileBuffer.remaining()) { fileBuffer.put(bb); } else { input.pushBack(value); break; } } int size = fileBuffer.position(); return size; }
private void flushBuffer() throws BufferOverflowException { if (pagedIn) { MappedByteBuffer buffer; // Only flush if page was changed if (dirtyPage) { Collection<byte[]> values = pageMap.values(); buffer = spillFile.getPage(pageIndex); buffer.clear(); // number of elements buffer.putInt(values.size()); for (byte[] value : values) { // element length buffer.putInt(value.length); // element buffer.put(value, 0, value.length); } } buffer = null; // Reset page stats pageMap.clear(); totalResultSize = 0; } pagedIn = false; dirtyPage = false; }
private void flushBuffer() throws BufferOverflowException { if (pagedIn) { MappedByteBuffer buffer; // Only flush if page was changed if (dirtyPage) { Collection<byte[]> values = pageMap.values(); buffer = spillFile.getPage(pageIndex); buffer.clear(); // number of elements buffer.putInt(values.size()); for (byte[] value : values) { // element length buffer.putInt(value.length); // element buffer.put(value, 0, value.length); } } buffer = null; // Reset page stats pageMap.clear(); totalResultSize = 0; } pagedIn = false; dirtyPage = false; }
/** * Resets this object to the post-configuration state. */ public void reset() { serializer[0] = null; serializer[1] = null; fileBuffer.clear(); }
protected DataInputBase(final MappedByteBuffer inputBuff) { this.inputBuff = inputBuff; inputBuff.clear(); }
protected DataInputBase(final MappedByteBuffer inputBuff) { this.inputBuff = inputBuff; inputBuff.clear(); }
@Override protected void seek(final long offset) throws IOException { buffer.clear().position((int) offset); }
@Override public void writeExternal(final ObjectOutput out) throws IOException { // write buffer capacity and data final byte buff[] = new byte[inputBuff.capacity()]; inputBuff.clear(); // reset the position inputBuff.get(buff); out.writeInt(buff.length); out.write(buff); }
/** * Close channel/connection to the file, clear buffer. */ public void close() { try { buffer.force(); buffer.clear(); fileChannel.close(); randomAccessFile.close(); this.bufferOffset = BEGIN; file.delete(); } catch (IOException ex) { LoggerFactory.getLogger(FileHandler.class.getName()).error("Close connection <" + fname + "> errors!", ex); } }
/** * Close channel/connection to the file, clear buffer. */ public void close() { try { buffer.force(); buffer.clear(); fileChannel.close(); randomAccessFile.close(); this.bufferOffset = BEGIN; file.delete(); } catch (IOException ex) { LogManager.getLogger(FileHandler.class.getName()).log(Level.ERROR, "Close connection <" + fname + "> errors!", ex); } }
/** * Modify partial data in a file. */ private boolean modifyFile(String fileName, int start, byte[] replaceBs) throws Exception { java.nio.channels.FileChannel channel; try (java.io.RandomAccessFile raf = new java.io.RandomAccessFile(fileName, "rw")) { channel = raf.getChannel(); java.nio.MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_WRITE, start, replaceBs.length); for (int i = 0; i < replaceBs.length; i++) { byte src = buffer.get(i); buffer.put(i, replaceBs[i]); } buffer.force(); buffer.clear(); channel.close(); } return true; }
@Override public void position(long newPosition) throws IOException { int pos = (int)(newPosition & BUFFER_MASK); int ind = (int)(newPosition >> BUFFER_BITS); for(int i = 0; i < ind; i++) { _mmapArray[i].position((int)BUFFER_MASK); } _mmapArray[ind].position(pos); for(int i = ind; i < _mmapArray.length; i++) { _mmapArray[i].clear(); } _currentPosition = newPosition; }
@Override public void position(long newPosition) throws IOException { int pos = (int)(newPosition & BUFFER_MASK); int ind = (int)(newPosition >> BUFFER_BITS); for(int i = 0; i < ind; i++) { _mmapArray[i].position((int)BUFFER_MASK); } _mmapArray[ind].position(pos); for(int i = ind; i < _mmapArray.length; i++) { _mmapArray[i].clear(); } _currentPosition = newPosition; }
@Override public void position(long newPosition) throws IOException { int pos = (int)(newPosition & BUFFER_MASK); int ind = (int)(newPosition >> BUFFER_BITS); for(int i = 0; i < ind; i++) { _mmapArray[i].position((int)BUFFER_MASK); } _mmapArray[ind].position(pos); for(int i = ind; i < _mmapArray.length; i++) { _mmapArray[i].clear(); } _currentPosition = newPosition; }
@Override public void position(long newPosition) throws IOException { int pos = (int)(newPosition & BUFFER_MASK); int ind = (int)(newPosition >> BUFFER_BITS); for(int i = 0; i < ind; i++) { _mmapArray[i].position((int)BUFFER_MASK); } _mmapArray[ind].position(pos); for(int i = ind; i < _mmapArray.length; i++) { _mmapArray[i].clear(); } _currentPosition = newPosition; }
public void close() throws ReadException { try { channel.close(); mbb.clear(); raf.close(); unmap(mbb); } catch (IOException e) { throw new ReadException(e); } catch (Exception e) { throw new ReadException(e); } } public void unmap(final Object buffer) throws Exception {
private void flushBuffer() throws BufferOverflowException { if (pagedIn) { MappedByteBuffer buffer; // Only flush if page was changed if (dirtyPage) { Collection<byte[]> values = pageMap.values(); buffer = spillFile.getPage(pageIndex); buffer.clear(); // number of elements buffer.putInt(values.size()); for (byte[] value : values) { // element length buffer.putInt(value.length); // element buffer.put(value, 0, value.length); } } buffer = null; // Reset page stats pageMap.clear(); totalResultSize = 0; } pagedIn = false; dirtyPage = false; }
/** * Writes a single record to the memory-mapped file. This method does NOT take care of synchronization. The user * must guarantee that the file may be written to before calling this method. This method essentially reserves the * whole buffer for one record. As such it imposes some performance restrictions and should only be used when * absolutely necessary. * * @param value record to send * @return size of the written buffer * @throws IOException */ public int sendRecord(Object value) throws IOException { fileBuffer.clear(); int group = 0; serializer[group] = getSerializer(value); ByteBuffer bb = serializer[group].serialize(value); if (bb.remaining() > MAPPED_FILE_SIZE) { throw new RuntimeException("Serialized object does not fit into a single buffer."); } fileBuffer.put(bb); int size = fileBuffer.position(); reset(); return size; }
static void generateData(final MappedByteBuffer byteLayer, final long seed) { final int ringBuffSize = byteLayer.capacity(), countWordBytes = Long.SIZE / Byte.SIZE, countWords = ringBuffSize / countWordBytes, countTailBytes = ringBuffSize % countWordBytes; long word = seed; int i; // 64-bit words byteLayer.clear(); for(i = 0; i < countWords; i ++) { byteLayer.putLong(word); word = MathUtil.xorShift(word); } // tail bytes final ByteBuffer tailBytes = ByteBuffer.allocate(countWordBytes); tailBytes.asLongBuffer().put(word).rewind(); for(i = 0; i < countTailBytes; i ++) { byteLayer.put(countWordBytes * countWords + i, tailBytes.get(i)); } } }
static void generateData(final MappedByteBuffer byteLayer, final long seed) { final int ringBuffSize = byteLayer.capacity(); final int countWordBytes = Long.SIZE / Byte.SIZE; final int countWords = ringBuffSize / countWordBytes; final int countTailBytes = ringBuffSize % countWordBytes; long word = seed; int i; // 64-bit words byteLayer.clear(); for (i = 0; i < countWords; i++) { byteLayer.putLong(word); word = MathUtil.xorShift(word); } // tail bytes final ByteBuffer tailBytes = ByteBuffer.allocate(countWordBytes); tailBytes.asLongBuffer().put(word).rewind(); for (i = 0; i < countTailBytes; i++) { byteLayer.put(countWordBytes * countWords + i, tailBytes.get(i)); } } }