private void generatePartitionIds() throws Exception { for (int writer = 0; writer < partitionIds.length; writer++) { partitionIds[writer] = new ResultPartitionID(); } }
@Override public void readFrom(ByteBuf buffer) throws IOException { // TODO Directly deserialize fromNetty's buffer int length = buffer.readInt(); ByteBuffer serializedEvent = ByteBuffer.allocate(length); buffer.readBytes(serializedEvent); serializedEvent.flip(); event = (TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader()); partitionId = new ResultPartitionID(IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); receiverId = InputChannelID.fromByteBuf(buffer); } }
@Override public void readFrom(ByteBuf buffer) { partitionId = new ResultPartitionID(IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); queueIndex = buffer.readInt(); receiverId = InputChannelID.fromByteBuf(buffer); }
/** Convert relative partition directory to ResultPartitionID according to generatePartitionRootPath() method. */ public static ResultPartitionID convertRelativeDirToResultPartitionID(String relativeDir) { if (!relativeDir.startsWith(PARTITION_DIR_PREFIX)) { return null; } String[] segments = relativeDir.substring(PARTITION_DIR_PREFIX.length()).split(SPLITTER); if (segments == null || segments.length != 2) { return null; } try { return new ResultPartitionID( new IntermediateResultPartitionID(DatatypeConverter.parseHexBinary(segments[1])), new ExecutionAttemptID(DatatypeConverter.parseHexBinary(segments[0]))); } catch (Exception e) { return null; } }
static AddCredit readFrom(ByteBuf buffer) { ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); int credit = buffer.readInt(); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new AddCredit(partitionId, credit, receiverId); }
static AddCredit readFrom(ByteBuf buffer) { ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); int credit = buffer.readInt(); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new AddCredit(partitionId, credit, receiverId); }
static AddCredit readFrom(ByteBuf buffer) { ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); int credit = buffer.readInt(); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new AddCredit(partitionId, credit, receiverId); }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); return new PartialInputChannelDeploymentDescriptor( result, partitionId, partitionConnectionInfo); } }
static PartitionRequest readFrom(ByteBuf buffer) { ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); int queueIndex = buffer.readInt(); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); int credit = buffer.readInt(); return new PartitionRequest(partitionId, queueIndex, receiverId, credit); }
static PartitionRequest readFrom(ByteBuf buffer) { ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); int queueIndex = buffer.readInt(); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); int credit = buffer.readInt(); return new PartitionRequest(partitionId, queueIndex, receiverId, credit); }
static PartitionRequest readFrom(ByteBuf buffer) { ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); int queueIndex = buffer.readInt(); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); int credit = buffer.readInt(); return new PartitionRequest(partitionId, queueIndex, receiverId, credit); }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
ResultPartitionID partitionId = new ResultPartitionID(desc.getPartitionId(), executionId); ResultPartitionType partitionType = desc.getPartitionType(); if (partitionType.isBlocking() && shuffleType == BlockingShuffleType.YARN) {
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException { // directly deserialize fromNetty's buffer int length = buffer.readInt(); ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length); // assume this event's content is read from the ByteBuf (positions are not shared!) buffer.readerIndex(buffer.readerIndex() + length); TaskEvent event = (TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader); ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new TaskEventRequest(event, partitionId, receiverId); } }
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException { // directly deserialize fromNetty's buffer int length = buffer.readInt(); ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length); // assume this event's content is read from the ByteBuf (positions are not shared!) buffer.readerIndex(buffer.readerIndex() + length); TaskEvent event = (TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader); ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new TaskEventRequest(event, partitionId, receiverId); } }
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException { // directly deserialize fromNetty's buffer int length = buffer.readInt(); ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length); // assume this event's content is read from the ByteBuf (positions are not shared!) buffer.readerIndex(buffer.readerIndex() + length); TaskEvent event = (TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader); ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new TaskEventRequest(event, partitionId, receiverId); } }
/** * Recover the pipelined result partition consume status after job master failover. * * @param resultId The intermediate data set id in the log. */ public void recoverResultPartitionStatus( IntermediateDataSetID resultId, TaskManagerLocation location) { IntermediateResultPartition partitionToRecover = null; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (irp.getIntermediateResult().getId().equals(resultId)) { partitionToRecover = irp; } } if (partitionToRecover == null) { throw new FlinkRuntimeException("Can not find the intermediate result " + resultId + " on " + getTaskNameWithSubtaskIndex()); } if (!(ExecutionState.RUNNING.equals(currentExecution.getState()) && partitionToRecover.getResultType().isPipelined())) { throw new FlinkRuntimeException("Invalid state " + currentExecution.getState() + " for " + getTaskNameWithSubtaskIndex()); } currentExecution.getTaskManagerLocationFuture().complete(location); scheduleOrUpdateConsumers(new ResultPartitionID(partitionToRecover.getPartitionId(), currentExecution.getAttemptId())); }
int i = 0; for (IntermediateResultPartitionID irp : ev.getProducedPartitions().keySet()) { resultPartitionIds[i++] = new ResultPartitionID(irp, ev.getCurrentExecutionAttempt().getAttemptId());