public int reloadSegment(@Nonnull String tableNameWithType, @Nonnull String segmentName) { LOGGER.info("Sending reload message for segment: {} in table: {}", segmentName, tableNameWithType); Criteria recipientCriteria = new Criteria(); recipientCriteria.setRecipientInstanceType(InstanceType.PARTICIPANT); recipientCriteria.setInstanceName("%"); recipientCriteria.setResource(tableNameWithType); recipientCriteria.setPartition(segmentName); recipientCriteria.setSessionSpecific(true); SegmentReloadMessage segmentReloadMessage = new SegmentReloadMessage(tableNameWithType, segmentName); ClusterMessagingService messagingService = _helixZkManager.getMessagingService(); // Infinite timeout on the recipient int timeoutMs = -1; int numMessagesSent = messagingService.send(recipientCriteria, segmentReloadMessage, null, timeoutMs); if (numMessagesSent > 0) { LOGGER.info("Sent {} reload messages for segment: {} in table: {}", numMessagesSent, segmentName, tableNameWithType); } else { LOGGER.warn("No reload message sent for segment: {} in table: {}", segmentName, tableNameWithType); } return numMessagesSent; }
/** * Check if a given row matches the specified criteria * @param criteria the criteria * @param row row of currently persisted data * @return true if it matches, false otherwise */ private boolean rowMatches(Criteria criteria, ZNRecordRow row) { String instanceName = normalizePattern(criteria.getInstanceName()); String resourceName = normalizePattern(criteria.getResource()); String partitionName = normalizePattern(criteria.getPartition()); String partitionState = normalizePattern(criteria.getPartitionState()); return stringMatches(instanceName, row.getMapSubKey()) && stringMatches(resourceName, row.getRecordId()) && stringMatches(partitionName, row.getMapKey()) && stringMatches(partitionState, row.getMapValue()); }
Criteria recipientCriteria = new Criteria(); recipientCriteria.setInstanceName("localhost_12919"); recipientCriteria.setRecipientInstanceType(InstanceType.PARTICIPANT); recipientCriteria.setSelfExcluded(true); recipientCriteria.setSelfExcluded(false); AssertJUnit.assertEquals(1, svc.send(recipientCriteria, template)); recipientCriteria.setSelfExcluded(false); recipientCriteria.setInstanceName("%"); recipientCriteria.setResource("DB"); recipientCriteria.setPartition("%"); AssertJUnit.assertEquals(200, svc.send(recipientCriteria, template)); recipientCriteria.setSelfExcluded(false); recipientCriteria.setInstanceName("*"); recipientCriteria.setResource("DB"); recipientCriteria.setPartition("*"); AssertJUnit.assertEquals(200, svc.send(recipientCriteria, template)); recipientCriteria.setSelfExcluded(false); recipientCriteria.setInstanceName("localhost%"); recipientCriteria.setResource("DB"); recipientCriteria.setPartition("%"); AssertJUnit.assertEquals(200, svc.send(recipientCriteria, template)); recipientCriteria.setSelfExcluded(true); recipientCriteria.setInstanceName("%"); recipientCriteria.setResource("DB");
private void syncSessionToController(HelixManager manager) { if (_lastSessionSyncTime == null || System.currentTimeMillis() - _lastSessionSyncTime > SESSION_SYNC_INTERVAL) { // > delay since last sync HelixDataAccessor accessor = manager.getHelixDataAccessor(); PropertyKey key = new Builder(manager.getClusterName()).controllerMessage(SESSION_SYNC); if (accessor.getProperty(key) == null) { LOG.info(String.format("Participant %s syncs session with controller", manager.getInstanceName())); Message msg = new Message(MessageType.PARTICIPANT_SESSION_CHANGE, SESSION_SYNC); msg.setSrcName(manager.getInstanceName()); msg.setTgtSessionId("*"); msg.setMsgState(MessageState.NEW); msg.setMsgId(SESSION_SYNC); Criteria cr = new Criteria(); cr.setRecipientInstanceType(InstanceType.CONTROLLER); cr.setSessionSpecific(false); manager.getMessagingService().send(cr, msg); _lastSessionSyncTime = System.currentTimeMillis(); } } }
private void syncSessionToController(HelixManager manager) { if (_lastSessionSyncTime == null || System.currentTimeMillis() - _lastSessionSyncTime > SESSION_SYNC_INTERVAL) { // > delay since last sync HelixDataAccessor accessor = manager.getHelixDataAccessor(); PropertyKey key = new Builder(manager.getClusterName()).controllerMessage(SESSION_SYNC); if (accessor.getProperty(key) == null) { LOG.info(String.format("Participant %s syncs session with controller", manager.getInstanceName())); Message msg = new Message(MessageType.PARTICIPANT_SESSION_CHANGE, SESSION_SYNC); msg.setSrcName(manager.getInstanceName()); msg.setTgtSessionId("*"); msg.setMsgState(MessageState.NEW); msg.setMsgId(SESSION_SYNC); Criteria cr = new Criteria(); cr.setRecipientInstanceType(InstanceType.CONTROLLER); cr.setSessionSpecific(false); manager.getMessagingService().send(cr, msg); _lastSessionSyncTime = System.currentTimeMillis(); } } }
public int reloadAllSegments(@Nonnull String tableNameWithType) { LOGGER.info("Sending reload message for table: {}", tableNameWithType); Criteria recipientCriteria = new Criteria(); recipientCriteria.setRecipientInstanceType(InstanceType.PARTICIPANT); recipientCriteria.setInstanceName("%"); recipientCriteria.setResource(tableNameWithType); recipientCriteria.setSessionSpecific(true); SegmentReloadMessage segmentReloadMessage = new SegmentReloadMessage(tableNameWithType, null); ClusterMessagingService messagingService = _helixZkManager.getMessagingService(); // Infinite timeout on the recipient int timeoutMs = -1; int numMessagesSent = messagingService.send(recipientCriteria, segmentReloadMessage, null, timeoutMs); if (numMessagesSent > 0) { LOGGER.info("Sent {} reload messages for table: {}", numMessagesSent, tableNameWithType); } else { LOGGER.warn("No reload message sent for table: {}", tableNameWithType); } return numMessagesSent; }
DefaultParticipantErrorMessageHandlerFactory.ACTIONKEY, ActionOnError.DISABLE_INSTANCE.toString()); Criteria recipientCriteria = new Criteria(); recipientCriteria.setRecipientInstanceType(InstanceType.CONTROLLER); recipientCriteria.setSessionSpecific(false); _participants[0].getMessagingService().send(recipientCriteria, errorMessage1); DefaultParticipantErrorMessageHandlerFactory.ACTIONKEY, ActionOnError.DISABLE_PARTITION.toString()); Criteria recipientCriteria2 = new Criteria(); recipientCriteria2.setRecipientInstanceType(InstanceType.CONTROLLER); recipientCriteria2.setSessionSpecific(false); _participants[1].getMessagingService().send(recipientCriteria2, errorMessage2);
resultRow.put("instanceName", !recipientCriteria.getInstanceName().equals("") ? (!Strings.isNullOrEmpty(row.getMapSubKey()) ? row.getMapSubKey() : row.getRecordId()) : ""); resultRow.put("resourceName", !recipientCriteria.getResource().equals("") ? row.getRecordId() : ""); resultRow.put("partitionName", !recipientCriteria.getPartition().equals("") ? row.getMapKey() : ""); resultRow.put("partitionState", !recipientCriteria.getPartitionState().equals("") ? row.getMapValue() : ""); selected.add(resultRow);
@VisibleForTesting void sendShutdownRequest() { Criteria criteria = new Criteria(); criteria.setInstanceName("%"); criteria.setResource("%"); criteria.setPartition("%"); criteria.setPartitionState("%"); criteria.setRecipientInstanceType(InstanceType.CONTROLLER); criteria.setSessionSpecific(true); Message shutdownRequest = new Message(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString().toLowerCase() + UUID.randomUUID().toString()); shutdownRequest.setMsgSubType(HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString()); shutdownRequest.setMsgState(Message.MessageState.NEW); shutdownRequest.setTgtSessionId("*"); int messagesSent = this.helixManager.getMessagingService().send(criteria, shutdownRequest); if (messagesSent == 0) { LOGGER.error(String.format("Failed to send the %s message to the controller", shutdownRequest.getMsgSubType())); } }
/** * Check if a given row matches the specified criteria * @param criteria the criteria * @param row row of currently persisted data * @return true if it matches, false otherwise */ private boolean rowMatches(Criteria criteria, ZNRecordRow row) { String instanceName = normalizePattern(criteria.getInstanceName()); String resourceName = normalizePattern(criteria.getResource()); String partitionName = normalizePattern(criteria.getPartition()); String partitionState = normalizePattern(criteria.getPartitionState()); return (stringMatches(instanceName, Strings.nullToEmpty(row.getMapSubKey())) || stringMatches(instanceName, Strings.nullToEmpty(row.getRecordId()))) && stringMatches(resourceName, Strings.nullToEmpty(row.getRecordId())) && stringMatches(partitionName, Strings.nullToEmpty(row.getMapKey())) && stringMatches(partitionState, Strings.nullToEmpty(row.getMapValue())); }
Criteria recipientCriteria = new Criteria(); recipientCriteria.setRecipientInstanceType(InstanceType.PARTICIPANT); recipientCriteria.setInstanceName("%"); recipientCriteria.setSessionSpecific(true); recipientCriteria.setResource(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); recipientCriteria.setDataSource(Criteria.DataSource.EXTERNALVIEW); recipientCriteria.setPartition(offlineTableName);
/** * Check if a given row matches the specified criteria * @param criteria the criteria * @param row row of currently persisted data * @return true if it matches, false otherwise */ private boolean rowMatches(Criteria criteria, ZNRecordRow row) { String instanceName = normalizePattern(criteria.getInstanceName()); String resourceName = normalizePattern(criteria.getResource()); String partitionName = normalizePattern(criteria.getPartition()); String partitionState = normalizePattern(criteria.getPartitionState()); return (stringMatches(instanceName, Strings.nullToEmpty(row.getMapSubKey())) || stringMatches(instanceName, Strings.nullToEmpty(row.getRecordId()))) && stringMatches(resourceName, Strings.nullToEmpty(row.getRecordId())) && stringMatches(partitionName, Strings.nullToEmpty(row.getMapKey())) && stringMatches(partitionState, Strings.nullToEmpty(row.getMapValue())); }
@VisibleForTesting public static void sendUserDefinedMessage(String messageSubType, String messageVal, String messageId, InstanceType instanceType, HelixManager helixManager, Logger logger) { Criteria criteria = new Criteria(); criteria.setInstanceName("%"); criteria.setResource("%"); criteria.setPartition("%"); criteria.setPartitionState("%"); criteria.setRecipientInstanceType(instanceType); criteria.setSessionSpecific(true); Message message = new Message(Message.MessageType.USER_DEFINE_MSG.toString(), messageId); message.setMsgSubType(messageSubType); message.setAttribute(Message.Attributes.INNER_MESSAGE, messageVal); message.setMsgState(Message.MessageState.NEW); message.setTgtSessionId("*"); int messagesSent = helixManager.getMessagingService().send(criteria, message); if (messagesSent == 0) { logger.error(String.format("Failed to send the %s message to the participants", message)); } } }
/** * Check if a given row matches the specified criteria * @param criteria the criteria * @param row row of currently persisted data * @return true if it matches, false otherwise */ private boolean rowMatches(Criteria criteria, ZNRecordRow row) { String instanceName = normalizePattern(criteria.getInstanceName()); String resourceName = normalizePattern(criteria.getResource()); String partitionName = normalizePattern(criteria.getPartition()); String partitionState = normalizePattern(criteria.getPartitionState()); return stringMatches(instanceName, row.getMapSubKey()) && stringMatches(resourceName, row.getRecordId()) && stringMatches(partitionName, row.getMapKey()) && stringMatches(partitionState, row.getMapValue()); }
@VisibleForTesting void sendTokenFileUpdatedMessage(InstanceType instanceType) { Criteria criteria = new Criteria(); criteria.setInstanceName("%"); criteria.setResource("%"); criteria.setPartition("%"); criteria.setPartitionState("%"); criteria.setRecipientInstanceType(instanceType); criteria.setSessionSpecific(true);
/** * Check if a given row matches the specified criteria * @param criteria the criteria * @param row row of currently persisted data * @return true if it matches, false otherwise */ private boolean rowMatches(Criteria criteria, ZNRecordRow row) { String instanceName = normalizePattern(criteria.getInstanceName()); String resourceName = normalizePattern(criteria.getResource()); String partitionName = normalizePattern(criteria.getPartition()); String partitionState = normalizePattern(criteria.getPartitionState()); return stringMatches(instanceName, row.getMapSubKey()) && stringMatches(resourceName, row.getRecordId()) && stringMatches(partitionName, row.getMapKey()) && stringMatches(partitionState, row.getMapValue()); }
@VisibleForTesting void sendShutdownRequest() { final Criteria criteria = new Criteria(); criteria.setInstanceName("%"); criteria.setResource("%"); criteria.setPartition("%"); criteria.setPartitionState("%"); criteria.setRecipientInstanceType(InstanceType.CONTROLLER); criteria.setSessionSpecific(true); final Message shutdownRequest = new Message(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString().toLowerCase() + UUID.randomUUID().toString()); shutdownRequest.setMsgSubType(HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString()); shutdownRequest.setMsgState(Message.MessageState.NEW); shutdownRequest.setTgtSessionId("*"); // Wait for 5 minutes final int timeout = 300000; // Send shutdown request to Cluster master, which will send shutdown request to workers // Upon receiving shutdown response from workers, master will shut itself down and call back shutdownASG() final int messagesSent = this.helixManager.getMessagingService().send(criteria, shutdownRequest, shutdownASG(),timeout); if (messagesSent == 0) { LOGGER.error(String.format("Failed to send the %s message to the controller", shutdownRequest.getMsgSubType())); } }
resultRow.put("instanceName", !recipientCriteria.getInstanceName().equals("") ? (!Strings.isNullOrEmpty(row.getMapSubKey()) ? row.getMapSubKey() : row.getRecordId()) : ""); resultRow.put("resourceName", !recipientCriteria.getResource().equals("") ? row.getRecordId() : ""); resultRow.put("partitionName", !recipientCriteria.getPartition().equals("") ? row.getMapKey() : ""); resultRow.put("partitionState", !recipientCriteria.getPartitionState().equals("") ? row.getMapValue() : ""); selected.add(resultRow);
new SegmentRefreshMessage(offlineTableName, segmentName, segmentZKMetadata.getCrc()); Criteria recipientCriteria = new Criteria(); recipientCriteria.setRecipientInstanceType(InstanceType.PARTICIPANT); recipientCriteria.setInstanceName("%"); recipientCriteria.setResource(offlineTableName); recipientCriteria.setPartition(segmentName); recipientCriteria.setSessionSpecific(true);
resultRow.put("instanceName", !recipientCriteria.getInstanceName().equals("") ? (!Strings.isNullOrEmpty(row.getMapSubKey()) ? row.getMapSubKey() : row.getRecordId()) : ""); resultRow.put("resourceName", !recipientCriteria.getResource().equals("") ? row.getRecordId() : ""); resultRow.put("partitionName", !recipientCriteria.getPartition().equals("") ? row.getMapKey() : ""); resultRow.put("partitionState", !recipientCriteria.getPartitionState().equals("") ? row.getMapValue() : ""); selected.add(resultRow);