@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
@Override public String fromBytes(byte[] bytes) { CoordinatorStreamMessage message = new CoordinatorStreamMessage(keySerde.fromBytes(bytes).toArray(), new HashMap<>()); return message.getKey(); }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Serde<List<?>> serde = new JsonSerde<>(); Object[] keyArray = serde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(coordinatorStreamMessage.getKey(), (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(coordinatorStreamMessage.getKey()); } } } } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else if (type.equalsIgnoreCase(SetConfig.TYPE)) { SetConfig setConfig = new SetConfig(message); return setConfig.getConfigValue(); } else if (type.equalsIgnoreCase(SetTaskModeMapping.TYPE)) { SetTaskModeMapping setTaskModeMapping = new SetTaskModeMapping(message); return String.valueOf(setTaskModeMapping.getTaskMode()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
public static CoordinatorStreamMessage deserializeCoordinatorStreamMessage(OutgoingMessageEnvelope msg) { JsonSerde<List<?>> keySerde = new JsonSerde<>(); Object[] keyArray = keySerde.fromBytes((byte[]) msg.getKey()).toArray(); JsonSerde<Map<String, Object>> msgSerde = new JsonSerde<>(); Map<String, Object> valueMap = msgSerde.fromBytes((byte[]) msg.getMessage()); return new CoordinatorStreamMessage(keyArray, valueMap); }
@Test public void testCoordinatorStreamMessageIsDelete() { CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[] {}, null); assertTrue(message.isDelete()); assertNull(message.getMessageMap()); }
@Test public void testCoordinatorStreamMessage() { CoordinatorStreamMessage message = new CoordinatorStreamMessage("source"); assertEquals("source", message.getSource()); assertEquals(CoordinatorStreamMessage.VERSION, message.getVersion()); assertNotNull(message.getUsername()); assertTrue(message.getTimestamp() > 0); assertTrue(!message.isDelete()); CoordinatorStreamMessage secondMessage = new CoordinatorStreamMessage(message.getKeyArray(), message.getMessageMap()); assertEquals(secondMessage, message); }