private void writeObject(ObjectOutputStream out) throws IOException { out.defaultWriteObject(); final int size = dataSet.size(); out.writeInt(size); if (size > 0) { DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(out); for (T element : dataSet){ serializer.serialize(element, wrapper); } } }
public void add(T value, TypeSerializer<T> serializer) throws IOException { try { ByteArrayOutputStream outStream = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(outStream); serializer.serialize(value, out); localValue.add(outStream.toByteArray()); } catch (IOException e) { throw new IOException("Failed to serialize value '" + value + '\'', e); } }
public static <T> byte[] serializeToByteArray(TypeSerializer<T> serializer, T record) throws IOException { if (record == null) { throw new NullPointerException("Record to serialize to byte array must not be null."); } ByteArrayOutputStream bos = new ByteArrayOutputStream(64); DataOutputViewStreamWrapper outputViewWrapper = new DataOutputViewStreamWrapper(bos); serializer.serialize(record, outputViewWrapper); return bos.toByteArray(); }
public FromElementsFunction(TypeSerializer<T> serializer, Iterable<T> elements) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos); int count = 0; try { for (T element : elements) { serializer.serialize(element, wrapper); count++; } } catch (Exception e) { throw new IOException("Serializing the source elements failed: " + e.getMessage(), e); } this.serializer = serializer; this.elementsSerialized = baos.toByteArray(); this.numElements = count; }
@Override public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) { outTypeSerializer = outTypeInfo.createSerializer(executionConfig); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); try { outTypeSerializer.serialize(initialValue, out); } catch (IOException ioe) { throw new RuntimeException("Unable to serialize initial value of type " + initialValue.getClass().getSimpleName() + " of fold operator.", ioe); } serializedInitialValue = baos.toByteArray(); }
@Override public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) { accSerializer = accTypeInformation.createSerializer(executionConfig); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); try { accSerializer.serialize(initialValue, out); } catch (IOException ioe) { throw new RuntimeException("Unable to serialize initial value of type " + initialValue.getClass().getSimpleName() + " of fold window function.", ioe); } serializedInitialValue = baos.toByteArray(); }
@Override public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) { // out type is not used, just use this for the execution config accSerializer = accTypeInformation.createSerializer(executionConfig); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); try { accSerializer.serialize(initialValue, out); } catch (IOException ioe) { throw new RuntimeException("Unable to serialize initial value of type " + initialValue.getClass().getSimpleName() + " of fold window function.", ioe); } serializedInitialValue = baos.toByteArray(); } }
@Override public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) { accSerializer = accTypeInformation.createSerializer(executionConfig); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); try { accSerializer.serialize(initialValue, out); } catch (IOException ioe) { throw new RuntimeException("Unable to serialize initial value of type " + initialValue.getClass().getSimpleName() + " of fold window function.", ioe); } serializedInitialValue = baos.toByteArray(); } }
@Override public void processElement(StreamRecord<IN> element) throws Exception { IN value = element.getValue(); // generate initial operator state if (out == null) { out = checkpointStorage.createTaskOwnedStateStream(); } serializer.serialize(value, new DataOutputViewStreamWrapper(out)); }
@Before public void setUp() throws Exception { PipedInputStream pipedInput = new PipedInputStream(32*1024*1024); this.in = new DataInputViewStreamWrapper(pipedInput); this.out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput)); }
@Before public void setUp() throws Exception { PipedInputStream pipeIn = new PipedInputStream(1024*1024); PipedOutputStream pipeOut = new PipedOutputStream(pipeIn); this.in = new DataInputViewStreamWrapper(pipeIn); this.out = new DataOutputViewStreamWrapper(pipeOut); }
@Override public byte[] serializeValue(Tuple3<Integer, Integer, String> element) { ByteArrayOutputStream by = new ByteArrayOutputStream(); DataOutputView out = new DataOutputViewStreamWrapper(by); try { ts.serialize(new Tuple2<>(element.f0, element.f1), out); } catch (IOException e) { throw new RuntimeException("Error" , e); } return by.toByteArray(); }
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockBasedOutput = new BlockBasedOutput(this.stream, (int) blockSize); this.outView = new DataOutputViewStreamWrapper(this.blockBasedOutput); }
@Before public void setup() throws Exception { in = new PipedInputStream(1000); out = new PipedOutputStream(in); mIn = new DataInputViewStreamWrapper(in); mOut = new DataOutputViewStreamWrapper(out); }
@Override public void run() { try { try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort()); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) { final TypeSerializer<Long> serializer = LongSerializer.INSTANCE; final Random rnd = new Random(seed); for (int i = 0; i < numElements; i++) { serializer.serialize(rnd.nextLong(), out); } } } catch (Throwable t) { error.set(t); } } };
@Before public void setUp() throws Exception { if (!foldingStateDesc.isSerializerInitialized()) { foldingStateDesc.initializeSerializerUnlessSet(new ExecutionConfig()); } ByteArrayOutputStream out = new ByteArrayOutputStream(); StringSerializer.INSTANCE.serialize("42", new DataOutputViewStreamWrapper(out)); foldingState = ImmutableFoldingState.createState( foldingStateDesc, out.toByteArray() ); }
@Before public void setUp() throws Exception { if (!aggrStateDesc.isSerializerInitialized()) { aggrStateDesc.initializeSerializerUnlessSet(new ExecutionConfig()); } final String initValue = "42"; ByteArrayOutputStream out = new ByteArrayOutputStream(); aggrStateDesc.getSerializer().serialize(initValue, new DataOutputViewStreamWrapper(out)); aggrState = ImmutableAggregatingState.createState( aggrStateDesc, out.toByteArray() ); }
/** * Count how many bytes would be written if all records were directly serialized. */ @Before public void calcRawDataSize() throws IOException { int recordIndex = 0; for (int fileIndex = 0; fileIndex < this.parallelism; fileIndex++) { ByteCounter byteCounter = new ByteCounter(); for (int fileCount = 0; fileCount < this.getNumberOfTuplesPerFile(fileIndex); fileCount++, recordIndex++) { writeRecord(this.getRecord(recordIndex), new DataOutputViewStreamWrapper(byteCounter)); } this.rawDataSizes[fileIndex] = byteCounter.getLength(); } }
@Test public void testReadNonVersioned() throws IOException { int preVersionedPayload = 563; TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable(); try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { restoredVersionedReadableWritable.read(in); } Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData()); }
@Test public void testReadMismatchVersion() throws Exception { String payload = "test"; TestWriteable testWriteable = new TestWriteable(1, payload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { testWriteable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } testWriteable = new TestWriteable(2); try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { testWriteable.read(new DataInputViewStreamWrapper(in)); Assert.fail("Version mismatch expected."); } catch (VersionMismatchException ignored) { } Assert.assertEquals(null, testWriteable.getData()); }