@Override public void readFieldsRequest(DataInput input) throws IOException { int numPartitions = input.readInt(); workerPartitions = new PairList<Integer, ExtendedDataOutput>(); workerPartitions.initialize(numPartitions); while (numPartitions-- > 0) { final int partitionId = input.readInt(); ExtendedDataOutput partitionData = WritableUtils.readExtendedDataOutput(input, getConf()); workerPartitions.add(partitionId, partitionData); } }
@Override public void writeRequest(DataOutput output) throws IOException { output.writeInt(workerPartitions.getSize()); PairList<Integer, ExtendedDataOutput>.Iterator iterator = workerPartitions.getIterator(); while (iterator.hasNext()) { iterator.next(); output.writeInt(iterator.getCurrentFirst()); WritableUtils.writeExtendedDataOutput( iterator.getCurrentSecond(), output); } }
/** * Gets all the data and removes it from the cache. * * @return All data for all vertices for all partitions */ public PairList<WorkerInfo, PairList<Integer, D>> removeAllData() { PairList<WorkerInfo, PairList<Integer, D>> allData = new PairList<WorkerInfo, PairList<Integer, D>>(); allData.initialize(dataSizes.length); for (WorkerInfo workerInfo : workerPartitions.keySet()) { PairList<Integer, D> workerData = removeWorkerData(workerInfo); if (!workerData.isEmpty()) { allData.add(workerInfo, workerData); } dataSizes[workerInfo.getTaskId()] = 0; } return allData; }
@Override public int getSerializedSize() { int size = super.getSerializedSize() + 4; PairList<Integer, B>.Iterator iterator = partitionVertexData.getIterator(); while (iterator.hasNext()) { iterator.next(); size += 4 + iterator.getCurrentSecond().getSerializedSize(); } return size; } }
/** * Check if the list is empty. * * @return True iff there are no pairs in the list */ public boolean isEmpty() { return getSize() == 0; }
@Override public int getSerializedSize() { // 4 for number of partitions int size = super.getSerializedSize() + 4; PairList<Integer, ExtendedDataOutput>.Iterator iterator = workerPartitions.getIterator(); while (iterator.hasNext()) { iterator.next(); // 4 bytes for the partition id and 4 bytes for the size size += 8 + iterator.getCurrentSecond().getPos(); } return size; } }
/** * Gets the data for a worker and removes it from the cache. * * @param workerInfo the address of the worker who owns the data * partitions that are receiving the data * @return List of pairs (partitionId, ByteArrayVertexIdData), * where all partition ids belong to workerInfo */ public PairList<Integer, D> removeWorkerData(WorkerInfo workerInfo) { PairList<Integer, D> workerData = new PairList<Integer, D>(); List<Integer> partitions = workerPartitions.get(workerInfo); workerData.initialize(partitions.size()); for (Integer partitionId : partitions) { if (dataCache[partitionId] != null) { workerData.add(partitionId, (D) dataCache[partitionId]); dataCache[partitionId] = null; } } dataSizes[workerInfo.getTaskId()] = 0; return workerData; }
@Override public void doRequest(ServerData<I, V, E> serverData) { PairList<Integer, ExtendedDataOutput>.Iterator iterator = workerPartitions.getIterator(); while (iterator.hasNext()) { iterator.next(); serverData.getPartitionStore() .addPartitionVertices(iterator.getCurrentFirst(), iterator.getCurrentSecond()); } }
@Override public void writeRequest(DataOutput output) throws IOException { output.writeInt(partitionVertexData.getSize()); PairList<Integer, B>.Iterator iterator = partitionVertexData.getIterator(); while (iterator.hasNext()) { iterator.next(); output.writeInt(iterator.getCurrentFirst()); iterator.getCurrentSecond().write(output); } }
@Override public void readFieldsRequest(DataInput input) throws IOException { int numPartitions = input.readInt(); partitionVertexData = new PairList<Integer, B>(); partitionVertexData.initialize(numPartitions); while (numPartitions-- > 0) { final int partitionId = input.readInt(); B vertexIdData = createVertexIdData(); vertexIdData.setConf(getConf()); vertexIdData.readFields(input); partitionVertexData.add(partitionId, vertexIdData); } }
@Override public void doRequest(ServerData serverData) { PairList<Integer, VertexIdEdges<I, E>>.Iterator iterator = partitionVertexData.getIterator(); while (iterator.hasNext()) { iterator.next(); serverData.getEdgeStore() .addPartitionEdges(iterator.getCurrentFirst(), iterator.getCurrentSecond()); } } }
/** * Gets all messages - vertexIds and removes them from the cache. * * @return All vertex messages for all workers */ private PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>> removeAllMsgVids() { PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>> allData = new PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>>(); allData.initialize(msgVidsCache.length); for (WorkerInfo workerInfo : getWorkerPartitions().keySet()) { ByteArrayOneMessageToManyIds<I, M> workerData = removeWorkerMsgVids(workerInfo); if (workerData != null && !workerData.isEmpty()) { allData.add(workerInfo, workerData); } } return allData; }
@Override public void doRequest(ServerData serverData) { PairList<Integer, VertexIdMessages<I, M>>.Iterator iterator = partitionVertexData.getIterator(); while (iterator.hasNext()) { iterator.next(); serverData.getIncomingMessageStore(). addPartitionMessages(iterator.getCurrentFirst(), iterator.getCurrentSecond()); } } }
PairList<WorkerInfo, PairList<Integer, ExtendedDataOutput>>.Iterator vertexIterator = remainingVertexCache.getIterator(); while (vertexIterator.hasNext()) { vertexIterator.next(); PairList<WorkerInfo, PairList<Integer, VertexIdEdges<I, E>>>.Iterator edgeIterator = remainingEdgeCache.getIterator(); while (edgeIterator.hasNext()) { edgeIterator.next();
/** * Flush the rest of the messages to the workers. */ public void flush() { PairList<WorkerInfo, PairList<Integer, VertexIdMessages<I, M>>> remainingMessageCache = removeAllMessages(); PairList<WorkerInfo, PairList< Integer, VertexIdMessages<I, M>>>.Iterator iterator = remainingMessageCache.getIterator(); while (iterator.hasNext()) { iterator.next(); WritableRequest writableRequest = new SendWorkerMessagesRequest<I, M>( iterator.getCurrentSecond()); totalMsgBytesSentInSuperstep += writableRequest.getSerializedSize(); clientProcessor.doRequest( iterator.getCurrentFirst(), writableRequest); } }
@Override public void flush() { super.flush(); PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>> remainingMsgVidsCache = removeAllMsgVids(); PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>>.Iterator msgIdsIterator = remainingMsgVidsCache.getIterator(); while (msgIdsIterator.hasNext()) { msgIdsIterator.next(); WritableRequest writableRequest = new SendWorkerOneMessageToManyRequest<>( msgIdsIterator.getCurrentSecond(), getConf()); totalMsgBytesSentInSuperstep += writableRequest.getSerializedSize(); clientProcessor.doRequest( msgIdsIterator.getCurrentFirst(), writableRequest); } } }