NCShutdownHook(NodeControllerService nodeControllerService) { super("ShutdownHook-" + nodeControllerService.getId()); this.nodeControllerService = nodeControllerService; }
@Override public String toString() { return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"node\" : \"" + ncs.getId() + "\" \"jobId\" : \"" + joblet.getJobId() + "\", \"taskId\" : \"" + taskAttemptId + "\" }"; } }
@Override public String toString() { return getName() + ": [" + ncs.getId() + "[" + jobId + ":" + taskId + "]"; } }
@Override public void run() { try { ncs.getClusterController(ccId).notifyPingResponse(ncs.getId()); } catch (Exception e) { LOGGER.info("failed to respond to ping from cc {}", ccId, e); } } }
private HeartbeatManager(NodeControllerService ncs, CcConnection ccc, HeartbeatData hbData, InetSocketAddress ncAddress) { this.ncs = ncs; this.ccc = ccc; hbThread = new Thread(new HeartbeatTask(ncs.getId(), hbData, ccc.getClusterControllerService(), ccc.getNodeParameters().getHeartbeatPeriod(), ncAddress), ncs.getId() + "-Heartbeat"); hbThread.setPriority(Thread.MAX_PRIORITY); hbThread.setDaemon(true); ccId = ccc.getCcId(); }
@Override public void run() { try { ncs.removeActivityClusterGraph(deployedJobSpecId); } catch (HyracksException e) { try { ncs.getClusterController(ccId).notifyDeployedJobSpecFailure(deployedJobSpecId, ncs.getId()); } catch (Exception e1) { e1.printStackTrace(); } } }
public synchronized void advertisePartitionRequest(TaskAttemptId taId, Collection<PartitionId> pids, IPartitionCollector collector, PartitionState minState) throws Exception { for (PartitionId pid : pids) { partitionRequestMap.put(pid, collector); PartitionRequest req = new PartitionRequest(pid, nodeController.getId(), taId, minState); nodeController.getClusterController(jobId.getCcId()).registerPartitionRequest(req); } }
@Override protected void doRun() throws Exception { List<JobProfile> profiles = new ArrayList<>(); Map<JobId, Joblet> jobletMap = ncs.getJobletMap(); jobletMap.values().stream().filter(ji -> ji.getJobId().getCcId().equals(ccId)) .forEach(ji -> profiles.add(new JobProfile(ji.getJobId()))); for (JobProfile jProfile : profiles) { Joblet ji; JobletProfile jobletProfile = new JobletProfile(ncs.getId()); ji = jobletMap.get(jProfile.getJobId()); if (ji != null) { ji.dumpProfile(jobletProfile); jProfile.getJobletProfiles().put(ncs.getId(), jobletProfile); } } fv.setValue(profiles); } }
@Override public String toString() { return getName() + ": [" + ncs.getId() + "[" + task.getJoblet().getJobId() + ":" + task.getTaskAttemptId() + "]"; } }
public NCMessageBroker(NodeControllerService ncs, MessagingProperties messagingProperties) { this.ncs = ncs; appContext = (INcApplicationContext) ncs.getApplicationContext(); maxMsgSize = messagingProperties.getFrameSize(); int messagingMemoryBudget = messagingProperties.getFrameSize() * messagingProperties.getFrameCount(); messagingFramePool = new ConcurrentFramePool(ncs.getId(), messagingMemoryBudget, messagingProperties.getFrameSize()); receivedMsgsQ = new LinkedBlockingQueue<>(); futureIdGenerator = new AtomicLong(); futureMap = new LongObjectHashMap<>(); MessageDeliveryService msgDeliverySvc = new MessageDeliveryService(); appContext.getThreadExecutor().execute(msgDeliverySvc); }
private void performCleanup() { nodeController.getJobletMap().remove(jobId); IJobletEventListener listener = getJobletEventListener(); if (listener != null) { listener.jobletFinish(cleanupStatus); } close(); cleanupPending = false; try { nodeController.getClusterController(jobId.getCcId()).notifyJobletCleanup(jobId, nodeController.getId()); } catch (Exception e) { e.printStackTrace(); } }
@Override public void run() { DeploymentStatus status; try { DeploymentUtils.deploy(deploymentId, binaryURLs, ncs.getContext().getJobSerializerDeserializerContainer(), ncs.getServerContext(), true); status = DeploymentStatus.SUCCEED; } catch (Exception e) { status = DeploymentStatus.FAIL; e.printStackTrace(); } try { IClusterController ccs = ncs.getClusterController(ccId); ccs.notifyDeployBinary(deploymentId, ncs.getId(), status); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void run() { DeploymentStatus status; try { DeploymentUtils.undeploy(deploymentId, ncs.getContext().getJobSerializerDeserializerContainer(), ncs.getServerContext()); status = DeploymentStatus.SUCCEED; } catch (Exception e) { status = DeploymentStatus.FAIL; } try { IClusterController ccs = ncs.getClusterController(ccId); ccs.notifyDeployBinary(deploymentId, ncs.getId(), status); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void run() { try { if (!upsert) { ncs.checkForDuplicateDeployedJobSpec(deployedJobSpecId); } ActivityClusterGraph acg = (ActivityClusterGraph) DeploymentUtils.deserialize(acgBytes, null, ncs.getContext()); ncs.storeActivityClusterGraph(deployedJobSpecId, acg); } catch (HyracksException e) { try { ncs.getClusterController(ccId).notifyDeployedJobSpecFailure(deployedJobSpecId, ncs.getId()); } catch (Exception e1) { e1.printStackTrace(); } } }
private void updatePartitionState(CcId ccId, PartitionId pid, TaskAttemptId taId, IPartition partition, PartitionState state) throws HyracksDataException { PartitionDescriptor desc = new PartitionDescriptor(pid, ncs.getId(), taId, partition.isReusable()); desc.setState(state); try { ncs.getClusterController(ccId).registerPartitionProvider(desc); } catch (Exception e) { throw HyracksDataException.create(e); } }
public static void send(CcId ccId, NodeControllerService cs, NodeStatus nodeStatus, SystemState systemState) throws HyracksDataException { try { RegistrationTasksRequestMessage msg = new RegistrationTasksRequestMessage(cs.getId(), nodeStatus, systemState); ((INCMessageBroker) cs.getContext().getMessageBroker()).sendMessageToCC(ccId, msg); } catch (Exception e) { LOGGER.log(Level.ERROR, "Unable to send RegistrationTasksRequestMessage to CC", e); throw HyracksDataException.create(e); } }
@BeforeClass public static void setUp() throws Exception { LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor); final NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs; final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>(); final String ip = InetAddress.getLoopbackAddress().getHostAddress(); for (NodeControllerService nc : ncs) { final String nodeId = nc.getId(); final INcApplicationContext appCtx = (INcApplicationContext) nc.getApplicationContext(); int apiPort = appCtx.getExternalProperties().getNcApiPort(); ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip, apiPort)); } testExecutor.setNcEndPoints(ncEndPoints); }
@Override public void run() { Exception ex = exceptions.get(0); LOGGER.log(ExceptionUtils.causedByInterrupt(ex) ? Level.DEBUG : Level.WARN, "task " + taskId + " has failed", ex); try { IResultPartitionManager resultPartitionManager = ncs.getResultPartitionManager(); if (resultPartitionManager != null) { resultPartitionManager.abortReader(jobId); } ncs.getClusterController(jobId.getCcId()).notifyTaskFailure(jobId, taskId, ncs.getId(), exceptions); } catch (Exception e) { LOGGER.log(Level.ERROR, "Failure reporting task failure to cluster controller", e); } if (task != null) { task.getJoblet().removeTask(task); } }
@Test public void nodeNotActiveTest() throws Exception { final String datasetName = "ds"; TestDataUtil.createIdOnlyDataset(datasetName); integrationUtil.ncs[0].setNodeStatus(NodeStatus.BOOTING); boolean thrown = false; try { testExecutor.executeSqlppUpdateOrDdl("select count(*) from " + datasetName + ";", TestCaseContext.OutputFormat.CLEAN_JSON); } catch (Exception e) { thrown = true; Assert.assertTrue(e.getMessage().contains(integrationUtil.ncs[0].getId())); } Assert.assertTrue(thrown); } }
@Override public void run() { TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId(), task.getPartitionSendProfile(), task.getStatsCollector()); try { ncs.getClusterController(task.getJobletContext().getJobId().getCcId()).notifyTaskComplete( task.getJobletContext().getJobId(), task.getTaskAttemptId(), ncs.getId(), taskProfile); } catch (Exception e) { LOGGER.log(Level.ERROR, "Failed notifying task complete for " + task.getTaskAttemptId(), e); } task.getJoblet().removeTask(task); }