@Setup(Level.Trial) public void createGenericIndexed() throws IOException { GenericIndexedWriter<byte[]> genericIndexedWriter = new GenericIndexedWriter<>( new OffHeapMemorySegmentWriteOutMedium(), "genericIndexedBenchmark", byteArrayStrategy ); genericIndexedWriter.open(); // GenericIndexObject caches prevObject for comparison, so need two arrays for correct objectsSorted computation. ByteBuffer[] elements = new ByteBuffer[2]; elements[0] = ByteBuffer.allocate(elementSize); elements[1] = ByteBuffer.allocate(elementSize); for (int i = 0; i < n; i++) { ByteBuffer element = elements[i & 1]; element.putInt(0, i); genericIndexedWriter.write(element.array()); } smooshDir = Files.createTempDir(); file = File.createTempFile("genericIndexedBenchmark", "meta"); try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); FileSmoosher fileSmoosher = new FileSmoosher(smooshDir)) { genericIndexedWriter.writeTo(fileChannel, fileSmoosher); } FileChannel fileChannel = FileChannel.open(file.toPath()); MappedByteBuffer byteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, file.length()); genericIndexed = GenericIndexed.read(byteBuffer, byteArrayStrategy, SmooshedFileMapper.load(smooshDir)); }
CompressedColumnarIntsSerializer( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int chunkFactor, final ByteOrder byteOrder, final CompressionStrategy compression ) { this( segmentWriteOutMedium, chunkFactor, byteOrder, compression, GenericIndexedWriter.ofCompressedByteBuffers( segmentWriteOutMedium, filenameBase, compression, chunkFactor * Integer.BYTES ) ); }
/** * Tries to get best value split(number of elements in each value file) which can be expressed as power of 2. * * @return Returns the size of value file splits as power of 2. * * @throws IOException */ private int bagSizePower() throws IOException { long avgObjectSize = (valuesOut.size() + numWritten - 1) / numWritten; for (int i = 31; i >= 0; --i) { if ((1L << i) * avgObjectSize <= fileSizeLimit) { if (actuallyFits(i)) { return i; } } } throw new ISE( "no value split found with fileSizeLimit [%d], avgObjectSize [%d]", fileSizeLimit, avgObjectSize ); }
@SuppressWarnings(value = "unchecked") @Override public void open() throws IOException { writer = new GenericIndexedWriter(segmentWriteOutMedium, StringUtils.format("%s.complex_column", filenameBase), strategy); writer.open(); }
static GenericIndexedWriter<ByteBuffer> ofCompressedByteBuffers( final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final CompressionStrategy compressionStrategy, final int bufferSize ) { GenericIndexedWriter<ByteBuffer> writer = new GenericIndexedWriter<>( segmentWriteOutMedium, filenameBase, compressedByteBuffersWriteObjectStrategy(compressionStrategy, bufferSize, segmentWriteOutMedium.getCloser()) ); writer.objectsSorted = false; return writer; }
private void writeEndBuffer() throws IOException { if (endBuffer != null) { endBuffer.flip(); if (endBuffer.remaining() > 0) { flattener.write(endBuffer); } endBuffer = null; } } }
@Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { writer.writeTo(channel, smoosher); } }
bitmapWriter = new GenericIndexedWriter<>( segmentWriteOutMedium, bmpFilename, indexSpec.getBitmapSerdeFactory().getObjectStrategy() ); bitmapWriter.open(); bitmapWriter.setObjectsNotSorted();
@Override public void open() throws IOException { flattener.open(); }
int bagSizePower = bagSizePower(); multiFileMetaSerdeHelper.writeTo(channel, this); .addWithSmooshedWriter(generateValueFileName(filenameBase, i), numBytesToPutInFile)) { writeBytesIntoSmooshedChannel(numBytesToPutInFile, buffer, smooshChannel, is); previousValuePosition = valuePosition; writeHeaderLong(smoosher, bagSizePower);
for (int i = 0; i < numberOfFilesRequired; i++) { ByteBuffer valueBuffer = fileMapper.mapFile(GenericIndexedWriter.generateValueFileName(columnName, i)); valueBuffersToUse[i] = valueBuffer.asReadOnlyBuffer(); ByteBuffer headerBuffer = fileMapper.mapFile(GenericIndexedWriter.generateHeaderFileName(columnName)); return new GenericIndexed<>( valueBuffersToUse,
static GenericIndexed<ResourceHolder<ByteBuffer>> ofCompressedByteBuffers( Iterable<ByteBuffer> buffers, CompressionStrategy compression, int bufferSize, ByteOrder order, Closer closer ) { return fromIterableVersionOne( buffers, GenericIndexedWriter.compressedByteBuffersWriteObjectStrategy(compression, bufferSize, closer), false, new DecompressingByteBufferObjectStrategy(order, compression) ); }
private void writeHeaderLong(FileSmoosher smoosher, int bagSizePower) throws IOException { ByteBuffer helperBuffer = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.nativeOrder()); int numberOfElementsPerValueFile = 1 << bagSizePower; long currentNumBytes = 0; long relativeRefBytes = 0; long relativeNumBytes; try (SmooshedWriter smooshChannel = smoosher .addWithSmooshedWriter(generateHeaderFileName(filenameBase), numWritten * Integer.BYTES)) { // following block converts long header indexes into int header indexes. for (int pos = 0; pos < numWritten; pos++) { //conversion of header offset from long to int completed for one value file done, change relativeRefBytes // to current offset. if ((pos & (numberOfElementsPerValueFile - 1)) == 0) { relativeRefBytes = currentNumBytes; } currentNumBytes = headerOutLong.getLong(pos); relativeNumBytes = currentNumBytes - relativeRefBytes; helperBuffer.putInt(0, Ints.checkedCast(relativeNumBytes)); helperBuffer.clear(); smooshChannel.write(helperBuffer); } } }
private void writeEndBuffer() throws IOException { if (endBuffer != null) { endBuffer.flip(); if (endBuffer.remaining() > 0) { flattener.write(endBuffer); } endBuffer = null; } } }
@SuppressWarnings(value = "unchecked") @Override public void open() throws IOException { writer = new GenericIndexedWriter( segmentWriteOutMedium, StringUtils.format("%s.complex_column", filenameBase), strategy, columnSize ); writer.open(); }
private void writeToVersionOne(WritableByteChannel channel) throws IOException { //null for the FileSmoosher means that we default to "version 1" of GenericIndexed. writer.writeTo(channel, null); } }
bitmapWriter = new GenericIndexedWriter<>( segmentWriteOutMedium, bmpFilename, indexSpec.getBitmapSerdeFactory().getObjectStrategy() ); bitmapWriter.open(); bitmapWriter.setObjectsNotSorted();
@Override public void open() throws IOException { flattener.open(); }
int bagSizePower = bagSizePower(); multiFileMetaSerdeHelper.writeTo(channel, this); .addWithSmooshedWriter(generateValueFileName(filenameBase, i), numBytesToPutInFile)) { writeBytesIntoSmooshedChannel(numBytesToPutInFile, buffer, smooshChannel, is); previousValuePosition = valuePosition; writeHeaderLong(smoosher, bagSizePower);