@Override public boolean isEmpty() { return extendedDataOutput.getPos() == 0; }
@Override protected int entrySerializedSize(ExtendedDataOutput vertices) { return vertices.getPos(); } }
/** * Get the number of bytes in the stream * * @return Number of bytes */ protected int getSize() { return dataOutput.getPos(); }
@Override public int getSize() { return extendedDataOutput.getPos(); }
/** * Write one of data outputs to another data output * * @param dataOutput Data output to write * @param out Data output to write to */ private void writeExtendedDataOutput(ExtendedDataOutput dataOutput, DataOutput out) throws IOException { out.writeInt(dataOutput.getPos()); out.write(dataOutput.getByteArray(), 0, dataOutput.getPos()); }
/** * Write ExtendedDataOutput to DataOutput * * @param extendedDataOutput ExtendedDataOutput to write * @param out DataOutput to write to */ public static void writeExtendedDataOutput( ExtendedDataOutput extendedDataOutput, DataOutput out) throws IOException { out.writeInt(extendedDataOutput.getPos()); out.write( extendedDataOutput.getByteArray(), 0, extendedDataOutput.getPos()); }
/** * Return threadLocal indexAndDataOutput instance * * @return threadLocal indexAndDataOutput instance */ public IndexAndDataOut getIndexAndDataOut() { IndexAndDataOut indexAndDataOut = threadLocal.get(); if (indexAndDataOut.dataOutput.getPos() >= threshold) { indexAndDataOut = newIndexAndDataOutput(); threadLocal.set(indexAndDataOut); } return indexAndDataOut; }
/** * Get DataOutput which data should be written to. If current DataOutput is * full it will create a new one. * * @return DataOutput which data should be written to */ private ExtendedDataOutput getDataOutputToWriteTo() { if (currentDataOutput.getPos() + SIZE_DELTA < MAX_SIZE) { return currentDataOutput; } else { if (dataOutputs == null) { dataOutputs = new ArrayList<ExtendedDataOutput>(1); } dataOutputs.add(currentDataOutput); currentDataOutput = conf.createExtendedDataOutput(MAX_SIZE); return currentDataOutput; } }
/** * Write a size of the message and message * * @param message Message to write */ private void writeMessageWithSize(M message) throws IOException { int pos = extendedDataOutput.getPos(); extendedDataOutput.skipBytes(4); writeData(extendedDataOutput, message); extendedDataOutput.writeInt( pos, extendedDataOutput.getPos() - pos - 4); }
@Override public ExtendedDataInput createDataInput() { return conf.createExtendedDataInput(dataOutput.getByteArray(), 0, dataOutput.getPos()); }
/** * Create extendedDataInput based on extendedDataOutput * * @param extendedDataOutput extendedDataOutput * @return extendedDataInput */ public ExtendedDataInput createExtendedDataInput( ExtendedDataOutput extendedDataOutput) { return createExtendedDataInput(extendedDataOutput.getByteArray(), 0, extendedDataOutput.getPos()); }
/** * Constructor. * * @param extendedDataOutput Extended data output * @param configuration Configuration */ public VertexIterator( ExtendedDataOutput extendedDataOutput, ImmutableClassesGiraphConfiguration<I, V, E> configuration) { extendedDataInput = configuration.createExtendedDataInput( extendedDataOutput.getByteArray(), 0, extendedDataOutput.getPos()); this.configuration = configuration; resetEmptyVertex(); }
@Override public void add(Edge<I, E> edge) { ExtendedDataOutput extendedDataOutput = getConf().createExtendedDataOutput( serializedEdges, serializedEdgesBytesUsed); try { WritableUtils.writeEdge(extendedDataOutput, edge); } catch (IOException e) { throw new IllegalStateException("add: Failed to write to the new " + "byte array"); } serializedEdges = extendedDataOutput.getByteArray(); serializedEdgesBytesUsed = extendedDataOutput.getPos(); ++edgeCount; }
@Override public void initialize(Iterable<Edge<I, E>> edges) { ExtendedDataOutput extendedOutputStream = getConf().createExtendedDataOutput(); for (Edge<I, E> edge : edges) { try { WritableUtils.writeEdge(extendedOutputStream, edge); } catch (IOException e) { throw new IllegalStateException("initialize: Failed to serialize " + edge); } ++edgeCount; } serializedEdges = extendedOutputStream.getByteArray(); serializedEdgesBytesUsed = extendedOutputStream.getPos(); }
/** * Constructor * * @param bigDataOutput {@link BigDataOutput} which we want to read data from */ public BigDataInput(BigDataOutput bigDataOutput) { dataInputs = new ArrayList<ExtendedDataInput>( bigDataOutput.getNumberOfDataOutputs()); for (ExtendedDataOutput dataOutput : bigDataOutput.getDataOutputs()) { dataInputs.add(bigDataOutput.getConf().createExtendedDataInput( dataOutput.getByteArray(), 0, dataOutput.getPos())); } currentPositionInInputs = -1; moveToNextDataInput(); }
@Override public M next() { long pointer = iterator.nextLong(); try { int index = (int) (pointer >>> 32); int offset = (int) pointer; ExtendedDataOutput buffer = msgBuffer.getDataOutput(index); messageReader.initialize(buffer.getByteArray(), offset, buffer.getPos()); reusableMsg.readFields(messageReader); } catch (IOException e) { throw new IllegalStateException("Got exception : " + e); } return reusableMsg; }
@Override public int getSerializedSize() { // 4 for number of partitions int size = super.getSerializedSize() + 4; PairList<Integer, ExtendedDataOutput>.Iterator iterator = workerPartitions.getIterator(); while (iterator.hasNext()) { iterator.next(); // 4 bytes for the partition id and 4 bytes for the size size += 8 + iterator.getCurrentSecond().getPos(); } return size; } }
/** * Write object to a byte array with the first 4 bytes as the size of the * entire buffer (including the size). * * @param writableObject Object to write from. * @param buffer Use this buffer instead * @param unsafe Use unsafe serialization? * @return Byte array with serialized object. */ public static byte[] writeToByteArrayWithSize(Writable writableObject, byte[] buffer, boolean unsafe) { ExtendedDataOutput extendedDataOutput; if (unsafe) { extendedDataOutput = new UnsafeByteArrayOutputStream(buffer); } else { extendedDataOutput = new ExtendedByteArrayDataOutput(buffer); } try { extendedDataOutput.writeInt(-1); writableObject.write(extendedDataOutput); extendedDataOutput.writeInt(0, extendedDataOutput.getPos()); } catch (IOException e) { throw new IllegalStateException("writeToByteArrayWithSize: " + "IOException", e); } return extendedDataOutput.getByteArray(); }
@Override public void addPartitionMessages( int partitionId, VertexIdMessages<I, M> messages) { try { VertexIdMessageIterator<I, M> vertexIdMessageIterator = messages.getVertexIdMessageIterator(); long pointer = 0; LongArrayList list; while (vertexIdMessageIterator.hasNext()) { vertexIdMessageIterator.next(); M msg = vertexIdMessageIterator.getCurrentMessage(); list = getOrCreateList(vertexIdMessageIterator); if (vertexIdMessageIterator.isNewMessage()) { IndexAndDataOut indexAndDataOut = bytesBuffer.getIndexAndDataOut(); pointer = indexAndDataOut.getIndex(); pointer <<= 32; ExtendedDataOutput dataOutput = indexAndDataOut.getDataOutput(); pointer += dataOutput.getPos(); msg.write(dataOutput); } synchronized (list) { list.add(pointer); } } } catch (IOException e) { throw new RuntimeException("addPartitionMessages: IOException while" + " adding messages for a partition: " + e); } }