/** * Cleans up and sets up store directories, validates changeLog SSPs for all stores of this task, * and registers SSPs with the respective consumers. */ public void initialize() { cleanBaseDirsAndReadOffsetFiles(); setupBaseDirs(); validateChangelogStreams(); getOldestChangeLogOffsets(); registerStartingOffsets(); }
public StoreProperties build() { return new StoreProperties(persistedToDisk, loggedStore); } }
public static void main(String[] args) { StateStorageTool tool = new StateStorageTool(); OptionSet options = tool.parser().parse(args); MapConfig config = tool.loadConfig(options); String path = tool.getPath(); StorageRecovery storageRecovery = new StorageRecovery(config, path); storageRecovery.run(); } }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
/** * setup phase which assigns required values to the variables used for all * tasks. */ private void setup() { log.info("setting up the recovery..."); getContainerModels(); getChangeLogSystemStreamsAndStorageFactories(); getChangeLogMaxPartitionNumber(); getContainerStorageManagers(); }
@Test public void testWriteOffsetFilesForNonPersistedStore() { final String storeName = "test-write-offset-non-persisted-store"; final String taskName = "test-write-offset-for-non-persisted-task"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, NON_LOGGED_STORE_DIR) .addInMemoryStore(storeName, ImmutableSet.of()) .build(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.writeOffsetFiles(); // should be no-op File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertFalse("Store directory: " + storeDir.getPath() + " should not be created for non-persisted store", storeDir.exists()); }
@Test public void testStop() { final String storeName = "test-stop-store"; final String taskName = "test-stop-task"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, NON_LOGGED_STORE_DIR) .addInMemoryStore(storeName, ImmutableSet.of()) .build(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.stop(); verify(testSideInputStorageManager.getStore(storeName)).stop(); verify(testSideInputStorageManager).writeOffsetFiles(); }
@Test public void testInit() { final String storeName = "test-init-store"; final String taskName = "test-init-task"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of()) .build(); initializeSideInputStorageManager(testSideInputStorageManager); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); }
@Override public StorageEngine getStorageEngine(String storeName, File storeDir, Serde<Object> keySerde, Serde<Object> msgSerde, MessageCollector collector, MetricsRegistry registry, SystemStreamPartition changeLogSystemStreamPartition, JobContext jobContext, ContainerContext containerContext, StoreMode storeMode) { StoreProperties storeProperties = new StoreProperties.StorePropertiesBuilder().setLoggedStore(true).build(); return new MockStorageEngine(storeName, storeDir, changeLogSystemStreamPartition, storeProperties); } }
@Before public void setup() throws InterruptedException { putConfig(); putMetadata(); }
TaskSideInputStorageManager build() { return spy(new TaskSideInputStorageManager(taskName, streamMetadataCache, storeBaseDir, stores, storeToProcessor, storeToSSps, systemAdmins, mock(Config.class), clock)); } }
public static void main(String[] args) { StateStorageTool tool = new StateStorageTool(); OptionSet options = tool.parser().parse(args); MapConfig config = tool.loadConfig(options); String path = tool.getPath(); StorageRecovery storageRecovery = new StorageRecovery(config, path); storageRecovery.run(); } }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
public StoreProperties build() { return new StoreProperties(persistedToDisk, loggedStore); } }
public static void main(String[] args) { StateStorageTool tool = new StateStorageTool(); OptionSet options = tool.parser().parse(args); MapConfig config = tool.loadConfig(options); String path = tool.getPath(); StorageRecovery storageRecovery = new StorageRecovery(config, path); storageRecovery.run(); } }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
public static void main(String[] args) { StateStorageTool tool = new StateStorageTool(); OptionSet options = tool.parser().parse(args); MapConfig config = tool.loadConfig(options); String path = tool.getPath(); StorageRecovery storageRecovery = new StorageRecovery(config, path); storageRecovery.run(); } }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
public static void main(String[] args) { StateStorageTool tool = new StateStorageTool(); OptionSet options = tool.parser().parse(args); MapConfig config = tool.loadConfig(options); String path = tool.getPath(); StorageRecovery storageRecovery = new StorageRecovery(config, path); storageRecovery.run(); } }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }