@Override public Long call() throws Exception { return _helixManager.isConnected() ? 1L : 0L; } });
protected boolean isConnected() { return _helixManager.isConnected(); }
protected boolean isConnected() { return managerClusterHelixManager.isConnected() && jobClusterHelixManager.isConnected(); }
@Override public ServiceStatus.Status getServiceStatus() { if (_isStarted) { // If we've connected to Helix at some point, the instance status depends on being connected to ZK if (helixManager.isConnected()) { return ServiceStatus.Status.GOOD; } else { return ServiceStatus.Status.BAD; } } // Return starting until zk is connected if (!helixManager.isConnected()) { return ServiceStatus.Status.STARTING; } else { _isStarted = true; _statusDescription = ServiceStatus.STATUS_DESCRIPTION_NONE; return ServiceStatus.Status.GOOD; } }
@VisibleForTesting boolean isHelixManagerConnected() { return this.helixManager.isPresent() && this.helixManager.get().isConnected(); }
private void checkHelixConnection(String opr, String flowName, String flowGroup) throws FlowConfigLoggedException { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized log.warn("System not yet initialized. Skipping operation " + opr); throw new FlowConfigLoggedException(HttpStatus.S_500_INTERNAL_SERVER_ERROR, "System not yet initialized. Skipping " + opr + " flowConfig [flowName=" + flowName + " flowGroup=" + flowGroup + "]"); } }
@VisibleForTesting void disconnectHelixManager() { if (this.helixManager.isConnected()) { this.helixManager.disconnect(); } }
@VisibleForTesting void disconnectHelixManager() { if (this.helixManager.isConnected()) { this.helixManager.disconnect(); } }
@VisibleForTesting void disconnectHelixManager() { if (this.jobHelixManager.isConnected()) { this.jobHelixManager.disconnect(); } if (this.taskDriverHelixManager.isPresent()) { this.taskDriverHelixManager.get().disconnect(); } }
protected void disconnect() { if (managerClusterHelixManager.isConnected()) { this.managerClusterHelixManager.disconnect(); } if (this.dedicatedManagerCluster) { if (jobClusterHelixManager.isConnected()) { this.jobClusterHelixManager.disconnect(); } if (taskDriverHelixManager.isPresent()) { this.taskDriverHelixManager.get().disconnect(); } if (jobClusterController.isPresent() && jobClusterController.get().isConnected()) { this.jobClusterController.get().disconnect(); } if (taskDriverClusterController.isPresent() && taskDriverClusterController.get().isConnected()) { this.taskDriverClusterController.get().disconnect(); } } }
/** {@inheritDoc} */ @Override public void onDeleteSpec(URI deletedSpecURI, String deletedSpecVersion, Properties headers) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Deletion: " + deletedSpecURI); return; } _log.info("Spec deletion detected: " + deletedSpecURI + "/" + deletedSpecVersion); try { Spec deletedSpec = this.scheduledFlowSpecs.get(deletedSpecURI.toString()); if (null != deletedSpec) { this.orchestrator.remove(deletedSpec, headers); this.scheduledFlowSpecs.remove(deletedSpecURI.toString()); unscheduleJob(deletedSpecURI.toString()); } else { _log.warn(String.format( "Spec with URI: %s was not found in cache. May be it was cleaned, if not please " + "clean it manually", deletedSpecURI)); } } catch (JobException e) { _log.warn(String.format("Spec with URI: %s was not unscheduled cleaning", deletedSpecURI), e); } }
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Update: " + updatedSpec); return; } _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof FlowSpec)) { return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
private static HelixManager createMockHelixManager(boolean isLeader, boolean isConnected) { HelixManager helixManager = mock(HelixManager.class); when(helixManager.isLeader()).thenReturn(isLeader); when(helixManager.isConnected()).thenReturn(isConnected); return helixManager; }
/** * Helix participant cannot pre-configure tags before it connects to ZK. So this method can only be invoked after * {@link HelixManager#connect()}. However this will still work because tagged jobs won't be sent to a non-tagged instance. Hence * the job with EXAMPLE_INSTANCE_TAG will remain in the ZK until an instance with EXAMPLE_INSTANCE_TAG was found. */ private void addInstanceTags() { List<String> tags = ConfigUtils.getStringList(this.config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY); HelixManager receiverManager = getReceiverManager(); if (receiverManager.isConnected()) { if (!tags.isEmpty()) { logger.info("Adding tags binding " + tags); tags.forEach(tag -> receiverManager.getClusterManagmentTool() .addInstanceTag(this.clusterName, this.helixInstanceName, tag)); logger.info("Actual tags binding " + receiverManager.getClusterManagmentTool() .getInstanceConfig(this.clusterName, this.helixInstanceName).getTags()); } } }
@AfterClass public void tearDown() throws IOException { try { if (this.helixManager.isConnected()) { this.helixManager.disconnect(); } this.localFs.delete(this.baseDir, true); } catch (Throwable t) { throw this.closer.rethrow(t); } finally { this.closer.close(); } }
@AfterClass public void tearDown() throws IOException, TimeoutException { try { this.gobblinAwsClusterLauncher.stop(); if (this.helixManager.isConnected()) { this.helixManager.disconnect(); } this.gobblinAwsClusterLauncher.disconnectHelixManager(); } finally { this.closer.close(); } }
@AfterClass public void tearDown() throws Exception { try { if (this.helixManager.isConnected()) { this.helixManager.disconnect(); } this.gobblinClusterManager.disconnectHelixManager(); } catch (Throwable t) { Assert.fail(); } finally { this.testingZKServer.close(); } }
@AfterClass public void tearDown() throws IOException, TimeoutException { try { Files.deleteIfExists(Paths.get(DYNAMIC_CONF_PATH)); Files.deleteIfExists(Paths.get(YARN_SITE_XML_PATH)); this.gobblinYarnAppLauncher.stopYarnClient(); if (this.helixManager.isConnected()) { this.helixManager.disconnect(); } this.gobblinYarnAppLauncher.disconnectHelixManager(); if (applicationId != null) { this.gobblinYarnAppLauncher.cleanUpAppWorkDirectory(applicationId); } } finally { this.closer.close(); } }
public SegmentCompletionProtocol.Response extendBuildTime(final SegmentCompletionProtocol.Request.Params reqParams) { if (!isLeader() || !_helixManager.isConnected()) { _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_NOT_LEADER, 1L); return SegmentCompletionProtocol.RESP_NOT_LEADER; } final String segmentNameStr = reqParams.getSegmentName(); final String instanceId = reqParams.getInstanceId(); final long offset = reqParams.getOffset(); final int extTimeSec = reqParams.getExtraTimeSec(); LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr); SegmentCompletionFSM fsm = null; SegmentCompletionProtocol.Response response = SegmentCompletionProtocol.RESP_FAILED; try { fsm = lookupOrCreateFsm(segmentName, SegmentCompletionProtocol.MSG_TYPE_COMMIT); response = fsm.extendBuildTime(instanceId, offset, extTimeSec); } catch (Exception e) { LOGGER.error("Caught exception in extendBuildTime for segment {}", segmentNameStr, e); } if (fsm != null && fsm.isDone()) { LOGGER.info("Removing FSM (if present):{}", fsm.toString()); _fsmMap.remove(segmentNameStr); } return response; }
/** * This method is to be called when a server calls in with the segmentConsumed() API, reporting an offset in the stream * that it currently has (i.e. next offset that it will consume, if it continues to consume). */ public SegmentCompletionProtocol.Response segmentConsumed(SegmentCompletionProtocol.Request.Params reqParams) { if (!isLeader() || !_helixManager.isConnected()) { _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_NOT_LEADER, 1L); return SegmentCompletionProtocol.RESP_NOT_LEADER; } final String segmentNameStr = reqParams.getSegmentName(); final String instanceId = reqParams.getInstanceId(); final String stopReason = reqParams.getReason(); final long offset = reqParams.getOffset(); LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr); SegmentCompletionProtocol.Response response = SegmentCompletionProtocol.RESP_FAILED; SegmentCompletionFSM fsm = null; try { fsm = lookupOrCreateFsm(segmentName, SegmentCompletionProtocol.MSG_TYPE_CONSUMED); response = fsm.segmentConsumed(instanceId, offset, stopReason); } catch (Exception e) { // Return failed response } if (fsm != null && fsm.isDone()) { LOGGER.info("Removing FSM (if present):{}", fsm.toString()); _fsmMap.remove(segmentNameStr); } return response; }