public TaskRestoreManager(TaskModel taskModel, Map<String, SystemStream> changelogSystemStreams, Map<String, StorageEngine> taskStores, SystemAdmins systemAdmins, Clock clock) { this.taskStores = taskStores; this.taskModel = taskModel; this.clock = clock; this.changelogSystemStreams = changelogSystemStreams; this.systemAdmins = systemAdmins; this.fileOffsets = new HashMap<>(); this.taskStoresToRestore = this.taskStores.entrySet().stream() .filter(x -> x.getValue().getStoreProperties().isLoggedStore()) .map(x -> x.getKey()).collect(Collectors.toSet()); }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
/** * Create stores' base directories for logged-stores if they dont exist. */ private void setupBaseDirs() { LOG.debug("Setting up base directories for stores."); taskStores.forEach((storeName, storageEngine) -> { if (storageEngine.getStoreProperties().isLoggedStore()) { File loggedStorePartitionDir = StorageManagerUtil.getStorePartitionDir(loggedStoreBaseDirectory, storeName, taskModel.getTaskName()); LOG.info("Using logged storage partition directory: " + loggedStorePartitionDir.toPath().toString() + " for store: " + storeName); if (!loggedStorePartitionDir.exists()) { loggedStorePartitionDir.mkdirs(); } } else { File nonLoggedStorePartitionDir = StorageManagerUtil.getStorePartitionDir(nonLoggedStoreBaseDirectory, storeName, taskModel.getTaskName()); LOG.info("Using non logged storage partition directory: " + nonLoggedStorePartitionDir.toPath().toString() + " for store: " + storeName); nonLoggedStorePartitionDir.mkdirs(); } }); }