.map(partitionId -> new InputChannelDeploymentDescriptor( partitionId, localMode ? ResultPartitionLocation.createLocal() : ResultPartitionLocation.createRemote(new ConnectionID(senderLocation, finalChannel)))) .toArray(InputChannelDeploymentDescriptor[]::new);
@Override public boolean equals(Object other) { if (other.getClass() != ConnectionID.class) { return false; } final ConnectionID ra = (ConnectionID) other; if (!ra.getAddress().equals(address) || ra.getConnectionIndex() != connectionIndex) { return false; } return true; }
@Override public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess()) { handInChannel(future.channel()); } else if (future.cause() != null) { notifyOfError(new RemoteTransportException( "Connecting to remote task manager + '" + connectionId.getAddress() + "' has failed. This might indicate that the remote task " + "manager has been lost.", connectionId.getAddress(), future.cause())); } else { notifyOfError(new LocalTransportException( "Connecting to remote task manager + '" + connectionId.getAddress() + "' has been cancelled.", null)); } } }
nettyClient.connect(connectionId.getAddress()).addListener(connectingChannel);
@Override public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { // use the yarn shuffle service data port Integer dataPort = configuration.getInteger( ExternalBlockShuffleServiceOptions.FLINK_SHUFFLE_SERVICE_PORT_KEY); // use the taskmanager ip address, for the shuffle service deployed on the same host // of the taskmanager is used to shuffle data to down streams. InetSocketAddress address = new InetSocketAddress(producerLocation.address(), dataPort); ConnectionID connectionId = new ConnectionID(address, intermediateResult.getConnectionIndex()); return ResultPartitionLocation.createRemote(connectionId); } }
@Override public boolean equals(Object other) { if (other.getClass() != ConnectionID.class) { return false; } final ConnectionID ra = (ConnectionID) other; if (!ra.getAddress().equals(address) || ra.getConnectionIndex() != connectionIndex) { return false; } return true; }
@Override public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess()) { handInChannel(future.channel()); } else if (future.cause() != null) { notifyOfError(new RemoteTransportException( "Connecting to remote task manager + '" + connectionId.getAddress() + "' has failed. This might indicate that the remote task " + "manager has been lost.", connectionId.getAddress(), future.cause())); } else { notifyOfError(new LocalTransportException( String.format( "Connecting to remote task manager '%s' has been cancelled.", connectionId.getAddress()), null)); } } }
@Override public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { if (consumerLocation.getResourceID().equals(producerLocation.getResourceID())) { // in the same taskmanager return ResultPartitionLocation.createLocal(); } else { // use the taskmanager ip and data port ConnectionID connectionId = new ConnectionID(producerLocation, intermediateResult.getConnectionIndex()); return ResultPartitionLocation.createRemote(connectionId); } } }
@Override public boolean equals(Object other) { if (other.getClass() != ConnectionID.class) { return false; } final ConnectionID ra = (ConnectionID) other; if (!ra.getAddress().equals(address) || ra.getConnectionIndex() != connectionIndex) { return false; } return true; }
@Override public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess()) { handInChannel(future.channel()); } else if (future.cause() != null) { notifyOfError(new RemoteTransportException( "Connecting to remote task manager + '" + connectionId.getAddress() + "' has failed. This might indicate that the remote task " + "manager has been lost.", connectionId.getAddress(), future.cause())); } else { notifyOfError(new LocalTransportException( String.format( "Connecting to remote task manager '%s' has been cancelled.", connectionId.getAddress()), null)); } } }
/** * Creates a channel deployment descriptor by completing the partition location. * * @see InputChannelDeploymentDescriptor */ public InputChannelDeploymentDescriptor createInputChannelDeploymentDescriptor(Execution consumerExecution) { checkNotNull(consumerExecution, "consumerExecution"); TaskManagerLocation consumerLocation = consumerExecution.getAssignedResourceLocation(); checkNotNull(consumerLocation, "Consumer connection info null"); final ResultPartitionLocation partitionLocation; if (consumerLocation.equals(partitionTaskManagerLocation)) { partitionLocation = ResultPartitionLocation.createLocal(); } else { partitionLocation = ResultPartitionLocation.createRemote( new ConnectionID(partitionTaskManagerLocation, partitionConnectionIndex)); } return new InputChannelDeploymentDescriptor(partitionID, partitionLocation); }
@Override public boolean equals(Object other) { if (other.getClass() != ConnectionID.class) { return false; } final ConnectionID ra = (ConnectionID) other; if (!ra.getAddress().equals(address) || ra.getConnectionIndex() != connectionIndex) { return false; } return true; }
@Override public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess()) { handInChannel(future.channel()); } else if (future.cause() != null) { notifyOfError(new RemoteTransportException( "Connecting to remote task manager + '" + connectionId.getAddress() + "' has failed. This might indicate that the remote task " + "manager has been lost.", connectionId.getAddress(), future.cause())); } else { notifyOfError(new LocalTransportException( "Connecting to remote task manager + '" + connectionId.getAddress() + "' has been cancelled.", null)); } } }
/** * Creates a channel deployment descriptor by completing the partition location. * * @see InputChannelDeploymentDescriptor */ public InputChannelDeploymentDescriptor createInputChannelDeploymentDescriptor(Execution consumerExecution) { checkNotNull(consumerExecution, "consumerExecution"); TaskManagerLocation consumerLocation = consumerExecution.getAssignedResourceLocation(); checkNotNull(consumerLocation, "Consumer connection info null"); final ResultPartitionLocation partitionLocation; if (consumerLocation.equals(partitionTaskManagerLocation)) { partitionLocation = ResultPartitionLocation.createLocal(); } else { partitionLocation = ResultPartitionLocation.createRemote( new ConnectionID(partitionTaskManagerLocation, partitionConnectionIndex)); } return new InputChannelDeploymentDescriptor(partitionID, partitionLocation); }
nettyClient.connect(connectionId.getAddress()).addListener(connectingChannel);
/** * Creates a channel deployment descriptor by completing the partition location. * * @see InputChannelDeploymentDescriptor */ public InputChannelDeploymentDescriptor createInputChannelDeploymentDescriptor(Execution consumerExecution) { checkNotNull(consumerExecution, "consumerExecution"); TaskManagerLocation consumerLocation = consumerExecution.getAssignedResourceLocation(); checkNotNull(consumerLocation, "Consumer connection info null"); final ResultPartitionLocation partitionLocation; if (consumerLocation.equals(partitionTaskManagerLocation)) { partitionLocation = ResultPartitionLocation.createLocal(); } else { partitionLocation = ResultPartitionLocation.createRemote( new ConnectionID(partitionTaskManagerLocation, partitionConnectionIndex)); } return new InputChannelDeploymentDescriptor(partitionID, partitionLocation); }
nettyClient.connect(connectionId.getAddress()).addListener(connectingChannel);
final ConnectionID connectionId = new ConnectionID( partitionTaskManagerLocation, partition.getIntermediateResult().getConnectionIndex());
nettyClient.connect(connectionId.getAddress()).addListener(connectingChannel);
final ConnectionID connectionId = new ConnectionID( partitionTaskManagerLocation, partition.getIntermediateResult().getConnectionIndex());