/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Gets the side input SSP offsets for all stores from their local offset files. * * @return a {@link Map} of {@link SystemStreamPartition} to offset in the offset files. */ @SuppressWarnings("unchecked") @VisibleForTesting Map<SystemStreamPartition, String> getFileOffsets() { LOG.info("Loading initial offsets from the file for side input stores."); Map<SystemStreamPartition, String> fileOffsets = new HashMap<>(); stores.keySet().forEach(storeName -> { LOG.debug("Reading local offsets for store: {}", storeName); File storeLocation = getStoreLocation(storeName); if (isValidSideInputStore(storeName, storeLocation)) { try { String fileContents = StorageManagerUtil.readOffsetFile(storeLocation, OFFSET_FILE); Map<SystemStreamPartition, String> offsets = OBJECT_MAPPER.readValue(fileContents, OFFSETS_TYPE_REFERENCE); fileOffsets.putAll(offsets); } catch (Exception e) { LOG.warn("Failed to load the offset file for side input store:" + storeName, e); } } }); return fileOffsets; }
/** * Gets the side input SSP offsets for all stores from their local offset files. * * @return a {@link Map} of {@link SystemStreamPartition} to offset in the offset files. */ @SuppressWarnings("unchecked") @VisibleForTesting Map<SystemStreamPartition, String> getFileOffsets() { LOG.info("Loading initial offsets from the file for side input stores."); Map<SystemStreamPartition, String> fileOffsets = new HashMap<>(); stores.keySet().forEach(storeName -> { LOG.debug("Reading local offsets for store: {}", storeName); File storeLocation = getStoreLocation(storeName); if (isValidSideInputStore(storeName, storeLocation)) { try { String fileContents = StorageManagerUtil.readOffsetFile(storeLocation, OFFSET_FILE); Map<SystemStreamPartition, String> offsets = OBJECT_MAPPER.readValue(fileContents, OFFSETS_TYPE_REFERENCE); fileOffsets.putAll(offsets); } catch (Exception e) { LOG.warn("Failed to load the offset file for side input store:" + storeName, e); } } }); return fileOffsets; }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Gets the side input SSP offsets for all stores from their local offset files. * * @return a {@link Map} of {@link SystemStreamPartition} to offset in the offset files. */ @SuppressWarnings("unchecked") @VisibleForTesting Map<SystemStreamPartition, String> getFileOffsets() { LOG.info("Loading initial offsets from the file for side input stores."); Map<SystemStreamPartition, String> fileOffsets = new HashMap<>(); stores.keySet().forEach(storeName -> { LOG.debug("Reading local offsets for store: {}", storeName); File storeLocation = getStoreLocation(storeName); if (isValidSideInputStore(storeName, storeLocation)) { try { String fileContents = StorageManagerUtil.readOffsetFile(storeLocation, OFFSET_FILE); Map<SystemStreamPartition, String> offsets = OBJECT_MAPPER.readValue(fileContents, OFFSETS_TYPE_REFERENCE); fileOffsets.putAll(offsets); } catch (Exception e) { LOG.warn("Failed to load the offset file for side input store:" + storeName, e); } } }); return fileOffsets; }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Gets the side input SSP offsets for all stores from their local offset files. * * @return a {@link Map} of {@link SystemStreamPartition} to offset in the offset files. */ @SuppressWarnings("unchecked") @VisibleForTesting Map<SystemStreamPartition, String> getFileOffsets() { LOG.info("Loading initial offsets from the file for side input stores."); Map<SystemStreamPartition, String> fileOffsets = new HashMap<>(); stores.keySet().forEach(storeName -> { LOG.debug("Reading local offsets for store: {}", storeName); File storeLocation = getStoreLocation(storeName); if (isValidSideInputStore(storeName, storeLocation)) { try { String fileContents = StorageManagerUtil.readOffsetFile(storeLocation, OFFSET_FILE); Map<SystemStreamPartition, String> offsets = OBJECT_MAPPER.readValue(fileContents, OFFSETS_TYPE_REFERENCE); fileOffsets.putAll(offsets); } catch (Exception e) { LOG.warn("Failed to load the offset file for side input store:" + storeName, e); } } }); return fileOffsets; }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Gets the side input SSP offsets for all stores from their local offset files. * * @return a {@link Map} of {@link SystemStreamPartition} to offset in the offset files. */ @SuppressWarnings("unchecked") @VisibleForTesting Map<SystemStreamPartition, String> getFileOffsets() { LOG.info("Loading initial offsets from the file for side input stores."); Map<SystemStreamPartition, String> fileOffsets = new HashMap<>(); stores.keySet().forEach(storeName -> { LOG.debug("Reading local offsets for store: {}", storeName); File storeLocation = getStoreLocation(storeName); if (isValidSideInputStore(storeName, storeLocation)) { try { String fileContents = StorageManagerUtil.readOffsetFile(storeLocation, OFFSET_FILE); Map<SystemStreamPartition, String> offsets = OBJECT_MAPPER.readValue(fileContents, OFFSETS_TYPE_REFERENCE); fileOffsets.putAll(offsets); } catch (Exception e) { LOG.warn("Failed to load the offset file for side input store:" + storeName, e); } } }); return fileOffsets; }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
@Test public void testInit() { final String storeName = "test-init-store"; final String taskName = "test-init-task"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of()) .build(); initializeSideInputStorageManager(testSideInputStorageManager); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); }
@Test public void testWriteOffsetFilesForNonPersistedStore() { final String storeName = "test-write-offset-non-persisted-store"; final String taskName = "test-write-offset-for-non-persisted-task"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, NON_LOGGED_STORE_DIR) .addInMemoryStore(storeName, ImmutableSet.of()) .build(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.writeOffsetFiles(); // should be no-op File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertFalse("Store directory: " + storeDir.getPath() + " should not be created for non-persisted store", storeDir.exists()); }
@Test public void testFlush() { final String storeName = "test-flush-store"; final String taskName = "test-flush-task"; final SystemStreamPartition ssp = new SystemStreamPartition("test-system", "test-stream", new Partition(0)); final String offset = "123"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of(ssp)) .build(); Map<String, StorageEngine> stores = new HashMap<>(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.updateLastProcessedOffset(ssp, offset); testSideInputStorageManager.flush(); for (StorageEngine storageEngine : stores.values()) { verify(storageEngine).flush(); } verify(testSideInputStorageManager).writeOffsetFiles(); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); Map<SystemStreamPartition, String> fileOffsets = testSideInputStorageManager.getFileOffsets(); assertTrue("Failed to get offset for ssp: " + ssp.toString() + " from file.", fileOffsets.containsKey(ssp)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp), offset); }
@Test public void testWriteOffsetFilesForPersistedStore() { final String storeName = "test-write-offset-persisted-store"; final String storeName2 = "test-write-offset-persisted-store-2"; final String taskName = "test-write-offset-for-persisted-task"; final String offset = "123"; final SystemStreamPartition ssp = new SystemStreamPartition("test-system", "test-stream", new Partition(0)); final SystemStreamPartition ssp2 = new SystemStreamPartition("test-system2", "test-stream2", new Partition(0)); TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of(ssp)) .addLoggedStore(storeName2, ImmutableSet.of(ssp2)) .build(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.updateLastProcessedOffset(ssp, offset); testSideInputStorageManager.updateLastProcessedOffset(ssp2, offset); testSideInputStorageManager.writeOffsetFiles(); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); Map<SystemStreamPartition, String> fileOffsets = testSideInputStorageManager.getFileOffsets(); assertTrue("Failed to get offset for ssp: " + ssp.toString() + " from file.", fileOffsets.containsKey(ssp)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp), offset); assertTrue("Failed to get offset for ssp: " + ssp2.toString() + " from file.", fileOffsets.containsKey(ssp2)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp2), offset); }