private MappingIterator<Entry<KeyType>> read(final File file, final Class<KeyType> keyClazz) { try { return spillMapper.readValues( spillMapper.getFactory().createParser(new LZ4BlockInputStream(new FileInputStream(file))), spillMapper.getTypeFactory().constructParametricType(Entry.class, keyClazz) ); } catch (IOException e) { throw Throwables.propagate(e); } }
@Test public void testSingleCharacterStream_singleByteRead() throws IOException { byte[] uncompressedData = new byte[] { SINGLE_VALUE }; initializeStreams(uncompressedData); int value = decompressingStream.read(); assertEquals(uncompressedData[0] & 0xFF, value); assertStreamIsEmpty(decompressingStream); }
@Override public int read() throws IOException { if (finished) { return -1; } if (o == originalLen) { refill(); } if (finished) { return -1; } return buffer[o++] & 0xFF; }
private void refill() throws IOException { readFully(compressedBuffer, HEADER_LENGTH); for (int i = 0; i < MAGIC_LENGTH; ++i) { if (compressedBuffer[i] != MAGIC[i]) { readFully(buffer, originalLen); break; case COMPRESSION_METHOD_LZ4: compressedBuffer = new byte[Math.max(compressedLen, compressedBuffer.length * 3 / 2)]; readFully(compressedBuffer, compressedLen); try { final int compressedLen2 = decompressor.decompress(compressedBuffer, 0, buffer, 0, originalLen);
@Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); }
@Override public int read() throws IOException { if (finished) { return -1; } if (o == originalLen) { refill(); } if (finished) { return -1; } return buffer[o++] & 0xFF; }
private void refill() throws IOException { readFully(compressedBuffer, HEADER_LENGTH); for (int i = 0; i < MAGIC_LENGTH; ++i) { if (compressedBuffer[i] != MAGIC[i]) { readFully(buffer, originalLen); break; case COMPRESSION_METHOD_LZ4: compressedBuffer = new byte[Math.max(compressedLen, compressedBuffer.length * 3 / 2)]; readFully(compressedBuffer, compressedLen); try { final int compressedLen2 = decompressor.decompress(compressedBuffer, 0, buffer, 0, originalLen);
private static DbFileSources.Data decodeRegularSourceData(byte[] binaryData) throws IOException { try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { return DbFileSources.Data.parseFrom(lz4Input); } catch (InvalidProtocolBufferException e) { if (SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE.equals(e.getMessage())) { return decodeHugeSourceData(binaryData); } throw e; } }
@Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); }
@Override public long skip(long n) throws IOException { if (finished) { return -1; } if (o == originalLen) { refill(); } if (finished) { return -1; } final int skipped = (int) Math.min(n, originalLen - o); o += skipped; return skipped; }
/** * Returns a dictionary of string keys added to this grouper. Note that the dictionary of keySerde is spilled on * local storage whenever the inner grouper is spilled. If there are spilled dictionaries, this method loads them * from disk and returns a merged dictionary. * * @return a dictionary which is a list of unique strings */ public List<String> mergeAndGetDictionary() { final Set<String> mergedDictionary = new HashSet<>(); mergedDictionary.addAll(keySerde.getDictionary()); for (File dictFile : dictionaryFiles) { try ( final MappingIterator<String> dictIterator = spillMapper.readValues( spillMapper.getFactory().createParser(new LZ4BlockInputStream(new FileInputStream(dictFile))), spillMapper.getTypeFactory().constructType(String.class) ) ) { while (dictIterator.hasNext()) { mergedDictionary.add(dictIterator.next()); } } catch (IOException e) { throw new RuntimeException(e); } } return new ArrayList<>(mergedDictionary); }
@Test public void testMultiBlock_singleByteReads() throws IOException { byte[] uncompressedData = new byte[16 * BLOCK_SIZE]; fillWithIncompressibleData(uncompressedData); initializeStreams(uncompressedData); for (int i = 0; i < uncompressedData.length; ++i) { int value = decompressingStream.read(); assertEquals(uncompressedData[i] & 0xFF, value); } assertStreamIsEmpty(decompressingStream); }
@Override public long skip(long n) throws IOException { if (finished) { return -1; } if (o == originalLen) { refill(); } if (finished) { return -1; } final int skipped = (int) Math.min(n, originalLen - o); o += skipped; return skipped; }
private static DbFileSources.Data decodeHugeSourceData(byte[] binaryData) throws IOException { try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { CodedInputStream input = CodedInputStream.newInstance(lz4Input); input.setSizeLimit(Integer.MAX_VALUE); return DbFileSources.Data.parseFrom(input); } }
@Override public int read(byte[] b, int off, int len) throws IOException { SafeUtils.checkRange(b, off, len); if (finished) { return -1; } if (o == originalLen) { refill(); } if (finished) { return -1; } len = Math.min(len, originalLen - o); System.arraycopy(buffer, o, b, off, len); o += len; return len; }
private void initializeStreams(byte[] uncompressedData) throws IOException { uncompressedStream = new ByteArrayInputStream(uncompressedData); compressingStream = new LZ4CompressingInputStream(uncompressedStream); decompressingStream = new LZ4BlockInputStream(compressingStream); }
@Override public int read(byte[] b, int off, int len) throws IOException { SafeUtils.checkRange(b, off, len); if (finished) { return -1; } if (o == originalLen) { refill(); } if (finished) { return -1; } len = Math.min(len, originalLen - o); System.arraycopy(buffer, o, b, off, len); o += len; return len; }
@Override public Map<Long, InputStream> loadStreams(Transaction t, Set<Long> ids) { Map<Long, InputStream> compressedStreams = super.loadStreams(t, ids); return Maps.transformValues(compressedStreams, stream -> { return new LZ4BlockInputStream(stream); }); }
@Override protected void tryWriteStreamToFile(Transaction transaction, Long id, StreamMetadata metadata, FileOutputStream fos) throws IOException { try (InputStream blockStream = makeStreamUsingTransaction(transaction, id, metadata); InputStream decompressingStream = new LZ4BlockInputStream(blockStream); OutputStream fileStream = fos;) { ByteStreams.copy(decompressingStream, fileStream); } }
@Override public InputStream loadStream(Transaction t, final Long id) { return new LZ4BlockInputStream(super.loadStream(t, id)); }