private <W extends Writable> W copyWritable(W writable) { if (reusedOut == null) { reusedOut = new UnsafeByteArrayOutputStream(); } if (reusedIn == null) { reusedIn = new UnsafeReusableByteArrayInput(); } return WritableUtils.createCopy(reusedOut, reusedIn, writable, null); }
/** * Construct LongsDiffWriter * @param useUnsafeWriter use unsafe writer */ public LongsDiffWriter(boolean useUnsafeWriter) { if (useUnsafeWriter) { out = new UnsafeByteArrayOutputStream(); } else { out = new ExtendedByteArrayDataOutput(); } }
@Override protected VertexIdEdges<I, E> readNextEntry(DataInput in) throws IOException { VertexIdEdges<I, E> vertexIdEdges = new ByteArrayVertexIdEdges<>(); vertexIdEdges.setConf(conf); vertexIdEdges.readFields(in); return vertexIdEdges; }
@Override public void writeRequest(DataOutput output) throws IOException { output.writeInt(workerPartitions.getSize()); PairList<Integer, ExtendedDataOutput>.Iterator iterator = workerPartitions.getIterator(); while (iterator.hasNext()) { iterator.next(); output.writeInt(iterator.getCurrentFirst()); WritableUtils.writeExtendedDataOutput( iterator.getCurrentSecond(), output); } }
@Override public void writeRequest(DataOutput output) throws IOException { output.writeInt(partitionVertexData.getSize()); PairList<Integer, B>.Iterator iterator = partitionVertexData.getIterator(); while (iterator.hasNext()) { iterator.next(); output.writeInt(iterator.getCurrentFirst()); iterator.getCurrentSecond().write(output); } }
@Override public int getSerializedSize() { int size = super.getSerializedSize() + 4; PairList<Integer, B>.Iterator iterator = partitionVertexData.getIterator(); while (iterator.hasNext()) { iterator.next(); size += 4 + iterator.getCurrentSecond().getSerializedSize(); } return size; } }
/** * Write a size of the message and message * * @param message Message to write */ private void writeMessageWithSize(M message) throws IOException { int pos = extendedDataOutput.getPos(); extendedDataOutput.skipBytes(4); writeData(extendedDataOutput, message); extendedDataOutput.writeInt( pos, extendedDataOutput.getPos() - pos - 4); }
/** * Construct LongsDiffReader * * @param compressedData Input byte array * @param useUnsafeReader use unsafe reader */ public LongsDiffReader(byte[] compressedData, boolean useUnsafeReader) { if (useUnsafeReader) { input = new UnsafeByteArrayInputStream(compressedData); } else { input = new ExtendedByteArrayDataInput(compressedData); } }
@Override public void add(byte[] serializedId, int idPos, M message) { if (!useMessageSizeEncoding) { super.add(serializedId, idPos, message); } else { try { extendedDataOutput.write(serializedId, 0, idPos); writeMessageWithSize(message); } catch (IOException e) { throw new IllegalStateException("add: IOException occurred"); } } }
@Override public void initialize() { super.initialize(); setUseMessageSizeEncoding(); }
/** * Wait for {@link ChannelFuture} to finish, while periodically * reporting progress. * * @param future ChannelFuture * @param progressable Progressable for reporting progress (Job context) */ public static void awaitChannelFuture(ChannelFuture future, Progressable progressable) { waitForever(new ChannelFutureWaitable(future), progressable); }
/** * Wait for {@link ChannelGroupFuture} to finish, while periodically * reporting progress. * * @param future ChannelGroupFuture * @param progressable Progressable for reporting progress (Job context) */ public static void awaitChannelGroupFuture(ChannelGroupFuture future, Progressable progressable) { waitForever(new ChannelGroupFutureWaitable(future), progressable); }
/** * Wait for executor tasks to terminate, while periodically reporting * progress. * * @param executor Executor which we are waiting for * @param progressable Progressable for reporting progress (Job context) */ public static void awaitExecutorTermination(ExecutorService executor, Progressable progressable) { waitForever(new ExecutorServiceWaitable(executor), progressable); }
@Override public void readFields(DataInput dataInput) throws IOException { extendedDataOutput = WritableUtils.readExtendedDataOutput(dataInput, getConf()); } }
public void postApplication() { for (Partition<I, V, E> partition : partitions) { for (Vertex<I, V, E> vertex : partition) { inputGraph.setVertex(vertex); } } } }
final Class<? extends Writable> getOutgoingMessage() { if (newMessage != null) { return newMessage; } if (newComputationClass == null) { return originalMessage; } Class[] computationTypes = ReflectionUtils.getTypeArguments( TypesHolder.class, newComputationClass); return computationTypes[4]; }
/** * Get number of vertices in the graph */ public int getVertexCount() { return testGraph.getVertexCount(); }
/** * Create an extended data output (can be subclassed) * * @return ExtendedDataOutput object */ public ExtendedDataOutput createExtendedDataOutput() { if (useUnsafeSerialization) { return new UnsafeByteArrayOutputStream(); } else { return new ExtendedByteArrayDataOutput(); } }
/** * Create an extended data input (can be subclassed) * * @param buf Buffer to use for the input * @return ExtendedDataInput object */ public ExtendedDataInput createExtendedDataInput(byte[] buf) { if (useUnsafeSerialization) { return new UnsafeByteArrayInputStream(buf); } else { return new ExtendedByteArrayDataInput(buf); } }
@Override public void initialize(int expectedSize) { super.initialize(expectedSize); setUseMessageSizeEncoding(); }