@Override public void run() { LOGGER.debug("cleaning up after job: {}", jobId); ncs.removeJobParameterByteStore(jobId); ncs.getPartitionManager().jobCompleted(jobId, status); Map<JobId, Joblet> jobletMap = ncs.getJobletMap(); Joblet joblet = jobletMap.remove(jobId); if (joblet != null) { joblet.cleanup(status); } } }
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new PipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId); } };
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor()); } };
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new MaterializedPartitionWriter(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor()); } };
private IPartitionCollector createPartitionCollector(TaskAttemptDescriptor td, final int partition, Task task, int i, IConnectorDescriptor conn, RecordDescriptor recordDesc, IConnectorPolicy cPolicy) throws HyracksDataException { IPartitionCollector collector = conn.createPartitionCollector(task, recordDesc, partition, td.getInputPartitionCounts()[i], td.getPartitionCount()); if (cPolicy.materializeOnReceiveSide()) { return new ReceiveSideMaterializingCollector(task, ncs.getPartitionManager(), collector, task.getTaskAttemptId(), ncs.getExecutor()); } else { return collector; } }
@Test public void failedJobPartitionRequestTest() throws Exception { final NodeControllerService nc1 = integrationUtil.ncs[0]; final NodeControllerService nc2 = integrationUtil.ncs[1]; final JobId failedJob = new JobId(-1); nc2.getPartitionManager().jobCompleted(failedJob, JobStatus.FAILURE); final NetworkAddress localNetworkAddress = nc2.getNetworkManager().getPublicNetworkAddress(); final InetSocketAddress nc2Address = new InetSocketAddress(localNetworkAddress.getAddress(), localNetworkAddress.getPort()); PartitionId id = new PartitionId(failedJob, new ConnectorDescriptorId(1), 0, 1); NetworkInputChannel inputChannel = new NetworkInputChannel(nc1.getNetworkManager(), nc2Address, id, 1); InputChannelFrameReader frameReader = new InputChannelFrameReader(inputChannel); inputChannel.registerMonitor(frameReader); AtomicBoolean failed = new AtomicBoolean(false); Thread reader = new Thread(() -> { try { failed.set(!frameReader.nextFrame(new FixedSizeFrame())); } catch (HyracksDataException e) { e.printStackTrace(); } }); reader.start(); final IHyracksCommonContext context = Mockito.mock(IHyracksCommonContext.class); Mockito.when(context.getInitialFrameSize()).thenReturn(2000); inputChannel.open(context); reader.join(5000); Assert.assertTrue(failed.get()); } }
final NetworkOutputChannel networkOutputChannel = new NetworkOutputChannel(ccb, 0); final MaterializingPipelinedPartition mpp = new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), pid, taId, ncs.getExecutor()); mpp.open();