/** * Read the taskName to partition mapping that is being maintained by this ChangelogManager * @return TaskName to change LOG partition mapping, or an empty map if there were no messages. */ public Map<TaskName, Integer> readPartitionMapping() { LOG.debug("Reading changelog partition information"); final HashMap<TaskName, Integer> changelogMapping = new HashMap<>(); for (CoordinatorStreamMessage coordinatorStreamMessage : coordinatorStreamManager.getBootstrappedStream(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapEntry = new SetChangelogMapping(coordinatorStreamMessage); changelogMapping.put(new TaskName(changelogMapEntry.getTaskName()), changelogMapEntry.getPartition()); LOG.debug("TaskName: {} is mapped to {}", changelogMapEntry.getTaskName(), changelogMapEntry.getPartition()); } return changelogMapping; }
public String getTaskName() { return getKey(); }
public int getPartition() { return Integer.parseInt(getMessageValue(CHANGELOG_VALUE_KEY)); } }
/** * The change log mapping message is used to store changelog partition information for a given task name. * @param source Source writing the change log mapping * @param taskName The task name to be used in the mapping * @param changelogPartitionNumber The partition to which the task's changelog is mapped to */ public SetChangelogMapping(String source, String taskName, int changelogPartitionNumber) { super(source); setType(TYPE); setKey(taskName); putMessageValue(CHANGELOG_VALUE_KEY, String.valueOf(changelogPartitionNumber)); }
@Override public byte[] toBytes(String value) { if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(SOURCE, "", value, "", ""); return messageSerde.toBytes(hostMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskContainerMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(SOURCE, "", Integer.valueOf(value)); return messageSerde.toBytes(changelogMapping.getMessageMap()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
private void convertConfigToCoordinatorMessage(Config config) { try { for (Map.Entry<String, String> configPair : config.entrySet()) { byte[] keyBytes = null; byte[] messgeBytes = null; if (configPair.getKey().startsWith(CHANGELOGPREFIX)) { String[] changelogInfo = configPair.getKey().split(":"); String changeLogPartition = configPair.getValue(); SetChangelogMapping changelogMapping = new SetChangelogMapping(changelogInfo[1], changelogInfo[2], Integer.parseInt(changeLogPartition)); keyBytes = MAPPER.writeValueAsString(changelogMapping.getKeyArray()).getBytes("UTF-8"); messgeBytes = MAPPER.writeValueAsString(changelogMapping.getMessageMap()).getBytes("UTF-8"); } else { SetConfig setConfig = new SetConfig("source", configPair.getKey(), configPair.getValue()); keyBytes = MAPPER.writeValueAsString(setConfig.getKeyArray()).getBytes("UTF-8"); messgeBytes = MAPPER.writeValueAsString(setConfig.getMessageMap()).getBytes("UTF-8"); } // The ssp here is the coordinator ssp (which is always fixed) and not the task ssp. put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, "", keyBytes, messgeBytes)); } setIsAtHead(systemStreamPartition, true); } catch (Exception e) { throw new SamzaException(e); } }
/** * Write the taskName to partition mapping. * @param changelogEntries The entries that needs to be written to the coordinator stream, the map takes the taskName * and it's corresponding changelog partition. */ public void writePartitionMapping(Map<TaskName, Integer> changelogEntries) { LOG.debug("Updating changelog information with: "); for (Map.Entry<TaskName, Integer> entry : changelogEntries.entrySet()) { LOG.debug("TaskName: {} to Partition: {}", entry.getKey().getTaskName(), entry.getValue()); coordinatorStreamManager.send(new SetChangelogMapping(SOURCE, entry.getKey().getTaskName(), entry.getValue())); } }
/** * The change log mapping message is used to store changelog partition information for a given task name. * @param source Source writing the change log mapping * @param taskName The task name to be used in the mapping * @param changelogPartitionNumber The partition to which the task's changelog is mapped to */ public SetChangelogMapping(String source, String taskName, int changelogPartitionNumber) { super(source); setType(TYPE); setKey(taskName); putMessageValue(CHANGELOG_VALUE_KEY, String.valueOf(changelogPartitionNumber)); }
@Override public byte[] toBytes(String value) { if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(SOURCE, "", value, "", ""); return messageSerde.toBytes(hostMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskContainerMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(SOURCE, "", Integer.valueOf(value)); return messageSerde.toBytes(changelogMapping.getMessageMap()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
/** * Write the taskName to partition mapping. * @param changelogEntries The entries that needs to be written to the coordinator stream, the map takes the taskName * and it's corresponding changelog partition. */ public void writePartitionMapping(Map<TaskName, Integer> changelogEntries) { LOG.debug("Updating changelog information with: "); for (Map.Entry<TaskName, Integer> entry : changelogEntries.entrySet()) { LOG.debug("TaskName: {} to Partition: {}", entry.getKey().getTaskName(), entry.getValue()); coordinatorStreamManager.send(new SetChangelogMapping(SOURCE, entry.getKey().getTaskName(), entry.getValue())); } }
/** * Read the taskName to partition mapping that is being maintained by this ChangelogManager * @return TaskName to change LOG partition mapping, or an empty map if there were no messages. */ public Map<TaskName, Integer> readPartitionMapping() { LOG.debug("Reading changelog partition information"); final HashMap<TaskName, Integer> changelogMapping = new HashMap<>(); for (CoordinatorStreamMessage coordinatorStreamMessage : coordinatorStreamManager.getBootstrappedStream(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapEntry = new SetChangelogMapping(coordinatorStreamMessage); changelogMapping.put(new TaskName(changelogMapEntry.getTaskName()), changelogMapEntry.getPartition()); LOG.debug("TaskName: {} is mapped to {}", changelogMapEntry.getTaskName(), changelogMapEntry.getPartition()); } return changelogMapping; }
/** * The change log mapping message is used to store changelog partition information for a given task name. * @param source Source writing the change log mapping * @param taskName The task name to be used in the mapping * @param changelogPartitionNumber The partition to which the task's changelog is mapped to */ public SetChangelogMapping(String source, String taskName, int changelogPartitionNumber) { super(source); setType(TYPE); setKey(taskName); putMessageValue(CHANGELOG_VALUE_KEY, String.valueOf(changelogPartitionNumber)); }
@Override public byte[] toBytes(String value) { if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(SOURCE, "", value, "", ""); return messageSerde.toBytes(hostMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskContainerMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(SOURCE, "", Integer.valueOf(value)); return messageSerde.toBytes(changelogMapping.getMessageMap()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } } }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
/** * Write the taskName to partition mapping. * @param changelogEntries The entries that needs to be written to the coordinator stream, the map takes the taskName * and it's corresponding changelog partition. */ public void writePartitionMapping(Map<TaskName, Integer> changelogEntries) { LOG.debug("Updating changelog information with: "); for (Map.Entry<TaskName, Integer> entry : changelogEntries.entrySet()) { LOG.debug("TaskName: {} to Partition: {}", entry.getKey().getTaskName(), entry.getValue()); coordinatorStreamManager.send(new SetChangelogMapping(SOURCE, entry.getKey().getTaskName(), entry.getValue())); } }
public int getPartition() { return Integer.parseInt(getMessageValue(CHANGELOG_VALUE_KEY)); } }
public String getTaskName() { return getKey(); }
/** * Read the taskName to partition mapping that is being maintained by this ChangelogManager * @return TaskName to change LOG partition mapping, or an empty map if there were no messages. */ public Map<TaskName, Integer> readPartitionMapping() { LOG.debug("Reading changelog partition information"); final HashMap<TaskName, Integer> changelogMapping = new HashMap<>(); for (CoordinatorStreamMessage coordinatorStreamMessage : coordinatorStreamManager.getBootstrappedStream(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapEntry = new SetChangelogMapping(coordinatorStreamMessage); changelogMapping.put(new TaskName(changelogMapEntry.getTaskName()), changelogMapEntry.getPartition()); LOG.debug("TaskName: {} is mapped to {}", changelogMapEntry.getTaskName(), changelogMapEntry.getPartition()); } return changelogMapping; }