/** * Updates data for dynamic optimization sent from Tasks. * @param dynOptData data used for data skew dynamic optimization. */ @Override public final void updateDynOptData(final Object dynOptData) { List<ControlMessage.PartitionSizeEntry> partitionSizeInfo = (List<ControlMessage.PartitionSizeEntry>) dynOptData; partitionSizeInfo.forEach(partitionSizeEntry -> { final Object key = partitionSizeEntry.getKey(); final long partitionSize = partitionSizeEntry.getSize(); if (aggregatedDynOptData.containsKey(key)) { aggregatedDynOptData.compute(key, (originalKey, originalValue) -> originalValue + partitionSize); } else { aggregatedDynOptData.put(key, partitionSize); } }); }
/** * <code>repeated .protobuf.PartitionSizeEntry partitionSize = 1;</code> */ public org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry.Builder addPartitionSizeBuilder() { return getPartitionSizeFieldBuilder().addBuilder( org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry.getDefaultInstance()); } /**
public int getSerializedSize() { int size = memoizedSerializedSize; if (size != -1) return size; size = 0; if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, getKeyBytes()); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(2, size_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; }
@Override public void emit(final O output) { final Map<Object, Long> aggregatedDynOptData = (Map<Object, Long>) output; final List<ControlMessage.PartitionSizeEntry> partitionSizeEntries = new ArrayList<>(); aggregatedDynOptData.forEach((key, size) -> partitionSizeEntries.add( ControlMessage.PartitionSizeEntry.newBuilder() // TODO #325: Add (de)serialization for non-string key types in data metric collection .setKey(key == null ? NULL_KEY : String.valueOf(key)) .setSize(size) .build()) ); connectionToMasterMap.getMessageSender(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID) .send(ControlMessage.Message.newBuilder() .setId(RuntimeIdManager.generateMessageId()) .setListenerId(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID) .setType(ControlMessage.MessageType.DataSizeMetric) .setDataSizeMetricMsg(ControlMessage.DataSizeMetricMsg.newBuilder() .addAllPartitionSize(partitionSizeEntries) ) .build()); // set the id of this vertex to mark the corresponding stage as put on hold taskExecutor.setIRVertexPutOnHold(irVertex); }
@Override public void emit(final O output) { final Map<Object, Long> aggregatedDynOptData = (Map<Object, Long>) output; final List<ControlMessage.PartitionSizeEntry> partitionSizeEntries = new ArrayList<>(); aggregatedDynOptData.forEach((key, size) -> partitionSizeEntries.add( ControlMessage.PartitionSizeEntry.newBuilder() .setKey(key == null ? NULL_KEY : String.valueOf(key)) .setSize(size) .build()) ); connectionToMasterMap.getMessageSender(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID) .send(ControlMessage.Message.newBuilder() .setId(RuntimeIdManager.generateMessageId()) .setListenerId(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID) .setType(ControlMessage.MessageType.DataSizeMetric) .setDataSizeMetricMsg(ControlMessage.DataSizeMetricMsg.newBuilder() .addAllPartitionSize(partitionSizeEntries) ) .build()); // set the id of this vertex to mark the corresponding stage as put on hold taskExecutor.setIRVertexPutOnHold(irVertex); }
/** * Updates data for dynamic optimization sent from Tasks. * @param dynOptData data used for data skew dynamic optimization. */ @Override public final void updateDynOptData(final Object dynOptData) { List<ControlMessage.PartitionSizeEntry> partitionSizeInfo = (List<ControlMessage.PartitionSizeEntry>) dynOptData; partitionSizeInfo.forEach(partitionSizeEntry -> { final Object key = partitionSizeEntry.getKey(); final long partitionSize = partitionSizeEntry.getSize(); if (aggregatedDynOptData.containsKey(key)) { aggregatedDynOptData.compute(key, (originalKey, originalValue) -> originalValue + partitionSize); } else { aggregatedDynOptData.put(key, partitionSize); } }); }
public org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry buildPartial() { org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry result = new org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) == 0x00000001)) { to_bitField0_ |= 0x00000001; } result.key_ = key_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { to_bitField0_ |= 0x00000002; } result.size_ = size_; result.bitField0_ = to_bitField0_; onBuilt(); return result; }
public Builder toBuilder() { return newBuilder(this); }
public org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry buildPartial() { org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry result = new org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) == 0x00000001)) { to_bitField0_ |= 0x00000001; } result.key_ = key_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { to_bitField0_ |= 0x00000002; } result.size_ = size_; result.bitField0_ = to_bitField0_; onBuilt(); return result; }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry prototype) {
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry prototype) {
public int getSerializedSize() { int size = memoizedSerializedSize; if (size != -1) return size; size = 0; if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, getKeyBytes()); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(2, size_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; }
public Builder toBuilder() { return newBuilder(this); }
public org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry getDefaultInstanceForType() { return org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry.getDefaultInstance(); }
public org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry getDefaultInstanceForType() { return org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry.getDefaultInstance(); }
public static Builder newBuilder(org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); }
public static Builder newBuilder(org.apache.nemo.runtime.common.comm.ControlMessage.PartitionSizeEntry prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); }
public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { getSerializedSize(); if (((bitField0_ & 0x00000001) == 0x00000001)) { output.writeBytes(1, getKeyBytes()); } if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeInt64(2, size_); } getUnknownFields().writeTo(output); }