/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
String key = null; try { SetConfig setConfigMessage = new SetConfig(message); key = setConfigMessage.getKey(); Map<String, String> valuesMap = (Map<String, String>) setConfigMessage.getMessageMap().get("values");
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else if (type.equalsIgnoreCase(SetConfig.TYPE)) { SetConfig setConfig = new SetConfig(message); return setConfig.getConfigValue(); } else if (type.equalsIgnoreCase(SetTaskModeMapping.TYPE)) { SetTaskModeMapping setTaskModeMapping = new SetTaskModeMapping(message); return String.valueOf(setTaskModeMapping.getTaskMode()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
configMap.remove(configKey); } else { String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue(); configMap.put(configKey, configValue);
@Test public void testHashCodeAndEquality() { SetConfig message = new SetConfig("source", "key1", "value1"); SetConfig message1 = new SetConfig("source", "key1", "value1"); SetConfig message2 = new SetConfig("source", "key2", "value1"); assertEquals(message.hashCode(), message1.hashCode()); assertEquals(message, message1); assertTrue(!message.equals(message2)); } }
@Override public byte[] toBytes(String value) { if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(SOURCE, "", value, "", ""); return messageSerde.toBytes(hostMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskContainerMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskModeMapping.TYPE)) { SetTaskModeMapping setTaskModeMapping = new SetTaskModeMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskModeMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(SOURCE, "", Integer.valueOf(value)); return messageSerde.toBytes(changelogMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetConfig.TYPE)) { SetConfig setConfig = new SetConfig(SOURCE, "", value); return messageSerde.toBytes(setConfig.getMessageMap()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } } }
private void convertConfigToCoordinatorMessage(Config config) { try { for (Map.Entry<String, String> configPair : config.entrySet()) { byte[] keyBytes = null; byte[] messgeBytes = null; if (configPair.getKey().startsWith(CHANGELOGPREFIX)) { String[] changelogInfo = configPair.getKey().split(":"); String changeLogPartition = configPair.getValue(); SetChangelogMapping changelogMapping = new SetChangelogMapping(changelogInfo[1], changelogInfo[2], Integer.parseInt(changeLogPartition)); keyBytes = MAPPER.writeValueAsString(changelogMapping.getKeyArray()).getBytes("UTF-8"); messgeBytes = MAPPER.writeValueAsString(changelogMapping.getMessageMap()).getBytes("UTF-8"); } else { SetConfig setConfig = new SetConfig("source", configPair.getKey(), configPair.getValue()); keyBytes = MAPPER.writeValueAsString(setConfig.getKeyArray()).getBytes("UTF-8"); messgeBytes = MAPPER.writeValueAsString(setConfig.getMessageMap()).getBytes("UTF-8"); } // The ssp here is the coordinator ssp (which is always fixed) and not the task ssp. put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, "", keyBytes, messgeBytes)); } setIsAtHead(systemStreamPartition, true); } catch (Exception e) { throw new SamzaException(e); } }
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> map = new LinkedHashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>(); assertEquals(1, systemStreamPartitions.size()); SystemStreamPartition systemStreamPartition = systemStreamPartitions.iterator().next(); assertEquals(expectedSystemStreamPartition, systemStreamPartition); if (pollCount++ == 0) { List<IncomingMessageEnvelope> list = new ArrayList<IncomingMessageEnvelope>(); SetConfig setConfig1 = new SetConfig("test", "job.name", "my-job-name"); SetConfig setConfig2 = new SetConfig("test", "job.id", "1234"); Delete delete = new Delete("test", "job.name", SetConfig.TYPE); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig1.getKeyArray()), serialize(setConfig1.getMessageMap()))); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig2.getKeyArray()), serialize(setConfig2.getMessageMap()))); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(delete.getKeyArray()), delete.getMessageMap())); map.put(systemStreamPartition, list); } return map; }
/** * Verify that if a particular key-value is written, then another, then the original again, * that the original occurs last in the set. */ @Test public void testOrderKeyRewrite() throws InterruptedException { final SystemStream systemStream = new SystemStream("system", "stream"); final SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0)); final SystemConsumer systemConsumer = mock(SystemConsumer.class); final List<IncomingMessageEnvelope> list = new ArrayList<>(); SetConfig setConfig1 = new SetConfig("source", "key1", "value1"); SetConfig setConfig2 = new SetConfig("source", "key1", "value2"); SetConfig setConfig3 = new SetConfig("source", "key1", "value1"); list.add(createIncomingMessageEnvelope(setConfig1, ssp)); list.add(createIncomingMessageEnvelope(setConfig2, ssp)); list.add(createIncomingMessageEnvelope(setConfig3, ssp)); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>() { { put(ssp, list); } }; when(systemConsumer.poll(anySet(), anyLong())).thenReturn(messages, Collections.<SystemStreamPartition, List<IncomingMessageEnvelope>>emptyMap()); CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin()); consumer.bootstrap(); Set<CoordinatorStreamMessage> bootstrappedMessages = consumer.getBoostrappedStream(); assertEquals(2, bootstrappedMessages.size()); // First message should have been removed as a duplicate CoordinatorStreamMessage[] coordinatorStreamMessages = bootstrappedMessages.toArray(new CoordinatorStreamMessage[2]); assertEquals(setConfig2, coordinatorStreamMessages[0]); assertEquals(setConfig3, coordinatorStreamMessages[1]); //Config 3 MUST be the last message, not config 2 }
@Test public void testSetConfig() { SetConfig setConfig = new SetConfig("source", "key", "value"); assertEquals(SetConfig.TYPE, setConfig.getType()); assertEquals("key", setConfig.getKey()); assertEquals("value", setConfig.getConfigValue()); assertFalse(setConfig.isDelete()); assertEquals(CoordinatorStreamMessage.VERSION, setConfig.getVersion()); }
MockSystemAdmin systemAdmin = new MockSystemAdmin(); CoordinatorStreamSystemProducer producer = new CoordinatorStreamSystemProducer(systemStream, systemProducer, systemAdmin); SetConfig setConfig1 = new SetConfig(source, "job.name", "my-job-name"); SetConfig setConfig2 = new SetConfig(source, "job.id", "1234"); Delete delete = new Delete(source, "job.name", SetConfig.TYPE); assertFalse(systemProducer.isRegistered());