/** * The SetConfig message is used to store a specific configuration. * @param message message to store which holds the configuration. */ public SetConfig(CoordinatorStreamMessage message) { super(message.getKeyArray(), message.getMessageMap()); }
/** * @param key * The key inside the messageMap, please only use human readable string (no JSON or such) - this allows * easy mutation of the coordinator stream outside of Samza (scripts) * @param value * The value corresponding to the key, should also be a simple string */ protected void putMessageValue(String key, String value) { getMessageValues().put(key, value); }
public CoordinatorStreamMessage(String source, Object[] keyArray, Map<String, Object> messageMap) { this(keyArray, messageMap); if (!isDelete) { this.messageMap.put("values", new HashMap<String, String>()); setSource(source); setUsername(System.getProperty("user.name")); setTimestamp(System.currentTimeMillis()); try { setHost(Util.getLocalHost().getHostAddress()); } catch (Exception e) { log.warn("Unable to retrieve host for current machine. Setting coordinator stream message host field to an empty string."); setHost(""); } } setVersion(VERSION); }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); log.debug("Received coordinator stream message: {}", coordinatorStreamMessage); if (SetConfig.TYPE.equals(coordinatorStreamMessage.getType())) { String configKey = coordinatorStreamMessage.getKey(); if (coordinatorStreamMessage.isDelete()) { configMap.remove(configKey); } else {
@Test public void testCoordinatorStreamMessage() { CoordinatorStreamMessage message = new CoordinatorStreamMessage("source"); assertEquals("source", message.getSource()); assertEquals(CoordinatorStreamMessage.VERSION, message.getVersion()); assertNotNull(message.getUsername()); assertTrue(message.getTimestamp() > 0); assertTrue(!message.isDelete()); CoordinatorStreamMessage secondMessage = new CoordinatorStreamMessage(message.getKeyArray(), message.getMessageMap()); assertEquals(secondMessage, message); }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
@Test public void testCoordinatorStreamMessageIsDelete() { CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[] {}, null); assertTrue(message.isDelete()); assertNull(message.getMessageMap()); }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Serde<List<?>> serde = new JsonSerde<>(); Object[] keyArray = serde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(coordinatorStreamMessage.getKey(), (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(coordinatorStreamMessage.getKey()); } } } } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
public Set<CoordinatorStreamMessage> getBootstrappedStream(String type) { log.debug("Bootstrapping coordinator stream for messages of type {}", type); bootstrap(); LinkedHashSet<CoordinatorStreamMessage> bootstrappedStream = new LinkedHashSet<CoordinatorStreamMessage>(); for (CoordinatorStreamMessage coordinatorStreamMessage : bootstrappedStreamSet) { log.trace("Considering message: {}", coordinatorStreamMessage); if (type.equalsIgnoreCase(coordinatorStreamMessage.getType())) { log.trace("Adding message: {}", coordinatorStreamMessage); bootstrappedStream.add(coordinatorStreamMessage); } } return bootstrappedStream; }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); log.debug("Received coordinator stream message: {}", coordinatorStreamMessage); if (SetConfig.TYPE.equals(coordinatorStreamMessage.getType())) { String configKey = coordinatorStreamMessage.getKey(); if (coordinatorStreamMessage.isDelete()) { configMap.remove(configKey); } else {
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
public Set<CoordinatorStreamMessage> getBootstrappedStream(String type) { log.debug("Bootstrapping coordinator stream for messages of type {}", type); bootstrap(); LinkedHashSet<CoordinatorStreamMessage> bootstrappedStream = new LinkedHashSet<CoordinatorStreamMessage>(); for (CoordinatorStreamMessage coordinatorStreamMessage : bootstrappedStreamSet) { log.trace("Considering message: {}", coordinatorStreamMessage); if (type.equalsIgnoreCase(coordinatorStreamMessage.getType())) { log.trace("Adding message: {}", coordinatorStreamMessage); bootstrappedStream.add(coordinatorStreamMessage); } } return bootstrappedStream; }
/** * SetTaskModeMapping is used to set the task to taskMode mapping information. * @param message which holds the mapped information. */ public SetTaskModeMapping(CoordinatorStreamMessage message) { super(message.getKeyArray(), message.getMessageMap()); }
public CoordinatorStreamMessage(String source, Object[] keyArray, Map<String, Object> messageMap) { this(keyArray, messageMap); if (!isDelete) { this.messageMap.put("values", new HashMap<String, String>()); setSource(source); setUsername(System.getProperty("user.name")); setTimestamp(System.currentTimeMillis()); try { setHost(Util.getLocalHost().getHostAddress()); } catch (Exception e) { log.warn("Unable to retrieve host for current machine. Setting coordinator stream message host field to an empty string."); setHost(""); } } setVersion(VERSION); }