/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Returns the offset with which the changelog consumer should be initialized for the given SystemStreamPartition. * * If a file offset exists, it represents the last changelog offset which is also reflected in the on-disk state. * In that case, we use the next offset after the file offset, as long as it is newer than the oldest offset * currently available in the stream. * * If there isn't a file offset or it's older than the oldest available offset, we simply start with the oldest. * * @param systemStreamPartition the changelog partition for which the offset is needed. * @param systemAdmin the [[SystemAdmin]] for the changelog. * @return the offset to from which the changelog consumer should be initialized. */ private String getStartingOffset(SystemStreamPartition systemStreamPartition, SystemAdmin systemAdmin) { String fileOffset = fileOffsets.get(systemStreamPartition); // NOTE: changeLogOldestOffsets may contain a null-offset for the given SSP (signifying an empty stream) // therefore, we need to differentiate that from the case where the offset is simply missing if (!changeLogOldestOffsets.containsKey(systemStreamPartition.getSystemStream())) { throw new SamzaException("Missing a change log offset for " + systemStreamPartition); } String oldestOffset = changeLogOldestOffsets.get(systemStreamPartition.getSystemStream()); return StorageManagerUtil.getStartingOffset(systemStreamPartition, systemAdmin, fileOffset, oldestOffset); }