@Test public void testRoundTrip() { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); writeType(sliceOutput, BOOLEAN); Type actualType = readType(new TestingTypeManager(), sliceOutput.slice().getInput()); assertEquals(actualType, BOOLEAN); } }
@Override public byte[] getSerializedData() { byte[] classifierBytes = serialize(classifier).getBytes(); DynamicSliceOutput output = new DynamicSliceOutput(classifierBytes.length + 64 * labelEnumeration.size()); output.appendInt(classifierBytes.length); output.appendBytes(classifierBytes); output.appendInt(labelEnumeration.size()); // Write the enumeration keys for (Map.Entry<Integer, String> entry : labelEnumeration.entrySet()) { output.appendInt(entry.getKey()); byte[] bytes = entry.getValue().getBytes(UTF_8); output.appendInt(bytes.length); output.appendBytes(bytes); } return output.slice().getBytes(); }
private static Block createVariableWidthBlock(int entries) { int[] offsets = new int[entries + 1]; DynamicSliceOutput dynamicSliceOutput = new DynamicSliceOutput(entries); for (int i = 0; i < entries; i++) { dynamicSliceOutput.writeByte(i); offsets[i + 1] = dynamicSliceOutput.size(); } return new VariableWidthBlock(entries, dynamicSliceOutput.slice(), offsets, Optional.empty()); } }
public Slice writePostscript(int footerLength, int metadataLength, CompressionKind compression, int compressionBlockSize) throws IOException { // postscript is not compressed DynamicSliceOutput output = new DynamicSliceOutput(64); metadataWriter.writePostscript(output, footerLength, metadataLength, compression, compressionBlockSize); return output.slice(); }
private static Slice decompressGzip(Slice input, int uncompressedSize) throws IOException { if (uncompressedSize == 0) { return EMPTY_SLICE; } DynamicSliceOutput sliceOutput = new DynamicSliceOutput(uncompressedSize); byte[] buffer = new byte[uncompressedSize]; try (InputStream gzipInputStream = new GZIPInputStream(input.getInput(), GZIP_BUFFER_SIZE)) { int bytesRead; while ((bytesRead = gzipInputStream.read(buffer)) != -1) { sliceOutput.write(buffer, 0, bytesRead); } return sliceOutput.getUnderlyingSlice(); } }
.writeTimeout(TIMEOUT_IN_MILLIS, TimeUnit.MILLISECONDS) .build(); slice = new DynamicSliceOutput(100); counter = new AtomicInteger(); cloudWatchClient = new AmazonCloudWatchAsyncClient(awsConfig.getCredentials()); generator.writeStartObject(); generator.writeFieldName("activities"); generator.writeStartArray(); byte[] base = (byte[]) slice.getUnderlyingSlice().getBase(); RequestBody body = RequestBody.create(mediaType, base, 0, slice.size()); } catch (Throwable e) { LOGGER.error(e, "Error while sending request to webhook"); slice.reset();
@Test public void testRetainedSize() throws Exception { int sliceOutputInstanceSize = ClassLayout.parseClass(DynamicSliceOutput.class).instanceSize(); DynamicSliceOutput output = new DynamicSliceOutput(10); long originalRetainedSize = output.getRetainedSize(); assertEquals(originalRetainedSize, sliceOutputInstanceSize + output.getUnderlyingSlice().getRetainedSize()); assertEquals(output.size(), 0); output.appendLong(0); output.appendShort(0); assertEquals(output.getRetainedSize(), originalRetainedSize); assertEquals(output.size(), 10); }
private void assertPositionValue(Block block, int position, Object expectedStackValue, long expectedHash, Object expectedObjectValue) assertEquals(objectValue, expectedObjectValue); if (objectValue != null) { assertInstanceOf(objectValue, objectValueType); assertEquals(hashPosition(type, block, position), expectedHash); fail("Expected UnsupportedOperationException"); assertEquals(block.isNull(position), expectedStackValue == null); if (block.isNull(position)) { if (type.isOrderable() && !(type instanceof UnknownType)) { Block nonNullValue = toBlock(getNonNullValue()); SliceOutput actualSliceOutput = new DynamicSliceOutput(100); writeBlock(blockEncodingSerde, actualSliceOutput, (Block) type.getObject(block, position)); SliceOutput expectedSliceOutput = new DynamicSliceOutput(actualSliceOutput.size()); writeBlock(blockEncodingSerde, expectedSliceOutput, (Block) expectedStackValue); assertEquals(actualSliceOutput.slice(), expectedSliceOutput.slice()); try { type.getBoolean(block, position);
private static int serializedSize(List<? extends Type> types, Page expectedPage) { PagesSerde serde = new TestingPagesSerdeFactory().createPagesSerde(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); writePages(serde, sliceOutput, expectedPage); Slice slice = sliceOutput.slice(); Iterator<Page> pageIterator = readPages(serde, slice.getInput()); if (pageIterator.hasNext()) { assertPageEquals(types, pageIterator.next(), expectedPage); } else { assertEquals(expectedPage.getPositionCount(), 0); } assertFalse(pageIterator.hasNext()); return slice.length(); } }
@Test public void testGrowCapacity() assertEquals(sliceOutput.getBufferCapacity(), 256); assertEquals(sliceOutput.getBufferCapacity(), 512); assertEquals(sliceOutput.getBufferCapacity(), 1200); DynamicSliceOutput output = new DynamicSliceOutput(6000); sliceOutput.close(); assertEquals(sliceOutput.writeDataTo(output), 200 + 200 + 1200 + 2000 + 2500);
@UsedByGeneratedCode public static Slice toJson(ObjectKeyProvider provider, JsonGeneratorWriter writer, ConnectorSession session, Block block) { try { Map<String, Integer> orderedKeyToValuePosition = new TreeMap<>(); for (int i = 0; i < block.getPositionCount(); i += 2) { String objectKey = provider.getObjectKey(block, i); orderedKeyToValuePosition.put(objectKey, i + 1); } SliceOutput output = new DynamicSliceOutput(40); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeStartObject(); for (Map.Entry<String, Integer> entry : orderedKeyToValuePosition.entrySet()) { jsonGenerator.writeFieldName(entry.getKey()); writer.writeJsonValue(jsonGenerator, block, entry.getValue(), session); } jsonGenerator.writeEndObject(); } return output.slice(); } catch (IOException e) { throwIfUnchecked(e); throw new RuntimeException(e); } } }
assertTrue(!request.getHeaders().get(PrestoHeaders.PRESTO_MAX_SIZE).isEmpty()); DataSize maxSize = DataSize.valueOf(request.getHeader(PrestoHeaders.PRESTO_MAX_SIZE)); assertEquals(maxSize, expectedMaxSize); HttpStatus status; if (!result.getSerializedPages().isEmpty()) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(64); PagesSerdeUtil.writeSerializedPages(sliceOutput, result.getSerializedPages()); bytes = sliceOutput.slice().getBytes(); status = HttpStatus.OK;
@Test public void testNoStartSync() throws Exception { SliceOutput output = new DynamicSliceOutput(10 * 1024); List<Segment> segments = ImmutableList.of( writeSegment(output, ImmutableList.of(ImmutableList.of(0, 2, 3, 4), ImmutableList.of(10, 12, 13))), writeSegment(output, ImmutableList.of(ImmutableList.of(20, 22), ImmutableList.of(30, 33), ImmutableList.of(40, 44))), writeSegment(output, ImmutableList.of(ImmutableList.of(100, 101, 102)))); assertFileSegments(output.slice(), segments); }
group.forEach(value -> writeValue(outputStream, value)); assertTrue(outputStream.getRetainedBytes() >= retainedBytes); retainedBytes = outputStream.getRetainedBytes(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1000); StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(33); streamDataOutput.writeData(sliceOutput); Stream stream = streamDataOutput.getStream(); assertEquals(stream.getStreamKind(), StreamKind.DATA); assertEquals(stream.getColumn(), 33); assertEquals(stream.getLength(), sliceOutput.size()); assertEquals(checkpoints.size(), groups.size()); R valueStream = createValueStream(sliceOutput.slice()); for (List<T> group : groups) { int index = 0;
@ScalarOperator(CAST) @LiteralParameters("x") @SqlType(JSON) public static Slice castFromVarchar(@SqlType("varchar(x)") Slice value) { try { SliceOutput output = new DynamicSliceOutput(value.length() + 2); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeString(value.toStringUtf8()); } return output.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%s' to %s", value.toStringUtf8(), JSON)); } }
public static Slice toJson(JsonGeneratorWriter writer, ConnectorSession session, Block block) { try { SliceOutput output = new DynamicSliceOutput(40); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeStartArray(); for (int i = 0; i < block.getPositionCount(); i++) { writer.writeJsonValue(jsonGenerator, block, i, session); } jsonGenerator.writeEndArray(); } return output.slice(); } catch (IOException e) { throwIfUnchecked(e); throw new RuntimeException(e); } } }
@Override public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { SliceOutput output = new DynamicSliceOutput(64); writeBlock(blockEncodingSerde, output, block); String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); jsonGenerator.writeString(encoded); } }
@ScalarOperator(CAST) @SqlType(JSON) public static Slice castFromDate(ConnectorSession session, @SqlType(DATE) long value) { try { SliceOutput output = new DynamicSliceOutput(12); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeString(printDate((int) value)); } return output.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%s' to %s", value, JSON)); } }
private static Slice decimalToJson(BigDecimal bigDecimal) { try { SliceOutput dynamicSliceOutput = new DynamicSliceOutput(32); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, dynamicSliceOutput)) { jsonGenerator.writeNumber(bigDecimal); } return dynamicSliceOutput.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%f' to %s", bigDecimal, StandardTypes.JSON)); } }
@ScalarFunction @LiteralParameters("x") @SqlType(StandardTypes.JSON) public static Slice jsonParse(@SqlType("varchar(x)") Slice slice) { // cast(json_parse(x) AS t)` will be optimized into `$internal$json_string_to_array/map/row_cast` in ExpressionOptimizer // If you make changes to this function (e.g. use parse JSON string into some internal representation), // make sure `$internal$json_string_to_array/map/row_cast` is changed accordingly. try (JsonParser parser = createJsonParser(JSON_FACTORY, slice)) { byte[] in = slice.getBytes(); SliceOutput dynamicSliceOutput = new DynamicSliceOutput(in.length); SORTED_MAPPER.writeValue((OutputStream) dynamicSliceOutput, SORTED_MAPPER.readValue(parser, Object.class)); // nextToken() returns null if the input is parsed correctly, // but will throw an exception if there are trailing characters. parser.nextToken(); return dynamicSliceOutput.slice(); } catch (Exception e) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Cannot convert '%s' to JSON", slice.toStringUtf8())); } }