@Override public void readFieldsRequest(DataInput input) throws IOException { int numPartitions = input.readInt(); workerPartitions = new PairList<Integer, ExtendedDataOutput>(); workerPartitions.initialize(numPartitions); while (numPartitions-- > 0) { final int partitionId = input.readInt(); ExtendedDataOutput partitionData = WritableUtils.readExtendedDataOutput(input, getConf()); workerPartitions.add(partitionId, partitionData); } }
/** * Gets the data for a worker and removes it from the cache. * * @param workerInfo the address of the worker who owns the data * partitions that are receiving the data * @return List of pairs (partitionId, ByteArrayVertexIdData), * where all partition ids belong to workerInfo */ public PairList<Integer, D> removeWorkerData(WorkerInfo workerInfo) { PairList<Integer, D> workerData = new PairList<Integer, D>(); List<Integer> partitions = workerPartitions.get(workerInfo); workerData.initialize(partitions.size()); for (Integer partitionId : partitions) { if (dataCache[partitionId] != null) { workerData.add(partitionId, (D) dataCache[partitionId]); dataCache[partitionId] = null; } } dataSizes[workerInfo.getTaskId()] = 0; return workerData; }
@Override public void readFieldsRequest(DataInput input) throws IOException { int numPartitions = input.readInt(); partitionVertexData = new PairList<Integer, B>(); partitionVertexData.initialize(numPartitions); while (numPartitions-- > 0) { final int partitionId = input.readInt(); B vertexIdData = createVertexIdData(); vertexIdData.setConf(getConf()); vertexIdData.readFields(input); partitionVertexData.add(partitionId, vertexIdData); } }
/** * Gets all the data and removes it from the cache. * * @return All data for all vertices for all partitions */ public PairList<WorkerInfo, PairList<Integer, D>> removeAllData() { PairList<WorkerInfo, PairList<Integer, D>> allData = new PairList<WorkerInfo, PairList<Integer, D>>(); allData.initialize(dataSizes.length); for (WorkerInfo workerInfo : workerPartitions.keySet()) { PairList<Integer, D> workerData = removeWorkerData(workerInfo); if (!workerData.isEmpty()) { allData.add(workerInfo, workerData); } dataSizes[workerInfo.getTaskId()] = 0; } return allData; }
/** * Gets all messages - vertexIds and removes them from the cache. * * @return All vertex messages for all workers */ private PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>> removeAllMsgVids() { PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>> allData = new PairList<WorkerInfo, ByteArrayOneMessageToManyIds<I, M>>(); allData.initialize(msgVidsCache.length); for (WorkerInfo workerInfo : getWorkerPartitions().keySet()) { ByteArrayOneMessageToManyIds<I, M> workerData = removeWorkerMsgVids(workerInfo); if (workerData != null && !workerData.isEmpty()) { allData.add(workerInfo, workerData); } } return allData; }