@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { // Add all job configuration properties so they are picked up by Hadoop Configuration conf = new Configuration(); for (Map.Entry<String, ConfigValue> entry : config.entrySet()) { conf.set(entry.getKey(), entry.getValue().unwrapped().toString()); } try { String stateStoreFsUri = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem stateStoreFs = FileSystem.get(URI.create(stateStoreFsUri), conf); String stateStoreRootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); return new FsStateStore(stateStoreFs, stateStoreRootDir, stateClass); } catch (IOException e) { throw new RuntimeException("Failed to create FsStateStore with factory", e); } } }
@Override public List<JobState.DatasetState> getAll(String storeName) throws IOException { return super.getAll(storeName); }
@Override public boolean create(String storeName, String tableName) throws IOException { Path storePath = new Path(this.storeRootDir, storeName); if (!this.fs.exists(storePath) && !create(storeName)) { return false; } Path tablePath = new Path(storePath, tableName); if (this.fs.exists(tablePath)) { throw new IOException(String.format("State file %s already exists for table %s", tablePath, tableName)); } return this.fs.createNewFile(tablePath); }
FsStateStore<WorkUnit> wuStateStore = new FsStateStore<>(this.localFs, workUnitDirPath.toString(), WorkUnit.class); Path workUnitFilePath = new Path(new Path(workUnitDirPath, TestHelper.TEST_JOB_ID), TestHelper.TEST_JOB_NAME + ".wu"); wuStateStore.put(TestHelper.TEST_JOB_ID, TestHelper.TEST_JOB_NAME + ".wu", workUnit);
public FsStateStore(FileSystem fs, String storeRootDir, Class<T> stateClass) { this.fs = fs; this.useTmpFileForPut = !FS_SCHEMES_NON_ATOMIC.contains(this.fs.getUri().getScheme()); this.conf = getConf(this.fs.getConf()); this.storeRootDir = storeRootDir; this.stateClass = stateClass; }
@Test public void testPutIntoTaskStateStore() throws IOException { TaskState taskState1 = new TaskState(); taskState1.setJobId(JOB_ID); taskState1.setTaskId(TASK_ID_0); this.taskStateStore.put(JOB_ID, TASK_ID_0 + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, taskState1); TaskState taskState2 = new TaskState(); taskState2.setJobId(JOB_ID); taskState2.setTaskId(TASK_ID_1); this.taskStateStore.put(JOB_ID, TASK_ID_1 + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, taskState2); }
public FsStateStore(String fsUri, String storeRootDir, Class<T> stateClass) throws IOException { this.conf = getConf(null); this.fs = FileSystem.get(URI.create(fsUri), this.conf); this.useTmpFileForPut = !FS_SCHEMES_NON_ATOMIC.contains(this.fs.getUri().getScheme()); this.storeRootDir = storeRootDir; this.stateClass = stateClass; }
@BeforeClass public void setUp() throws Exception { this.localFs = FileSystem.getLocal(new Configuration()); this.localFs.mkdirs(this.outputTaskStateDir); this.taskStateStore = new FsStateStore<>(this.localFs, this.outputTaskStateDir.toUri().getPath(), TaskState.class); this.taskStateCollectorService = new TaskStateCollectorService(new Properties(), this.jobState, this.eventBus, this.taskStateStore, new Path(this.outputTaskStateDir, JOB_ID)); this.eventBus.register(this); }
@Override public List<T> getAll(String storeName) throws IOException { List<T> states = Lists.newArrayList(); Path storePath = new Path(this.storeRootDir, storeName); if (!this.fs.exists(storePath)) { return states; } for (FileStatus status : this.fs.listStatus(storePath)) { states.addAll(getAll(storeName, status.getPath().getName())); } return states; }
public FsStateStore(String storeUrl, Class<T> stateClass) throws IOException { this.conf = getConf(null); Path storePath = new Path(storeUrl); this.fs = storePath.getFileSystem(this.conf); this.useTmpFileForPut = !FS_SCHEMES_NON_ATOMIC.contains(this.fs.getUri().getScheme()); this.storeRootDir = storePath.toUri().getPath(); this.stateClass = stateClass; }
Path tmpTablePath = new Path(new Path(this.storeRootDir, storeName), tmpTableName); if (!this.fs.exists(tmpTablePath) && !create(storeName, tmpTableName)) { throw new IOException("Failed to create a state file for table " + tmpTableName);
taskStateStore = new FsStateStore<>(this.fs, jobOutputPath.toString(), TaskState.class);
@Override public List<JobState.DatasetState> getAll(String storeName) throws IOException { return super.getAll(storeName); }
public FsStateStore(FileSystem fs, String storeRootDir, Class<T> stateClass) { this.fs = fs; this.useTmpFileForPut = !FS_SCHEMES_NON_ATOMIC.contains(this.fs.getUri().getScheme()); this.conf = getConf(this.fs.getConf()); this.storeRootDir = storeRootDir; this.stateClass = stateClass; }
Path tmpTablePath = new Path(new Path(this.storeRootDir, storeName), tmpTableName); if (!this.fs.exists(tmpTablePath) && !create(storeName, tmpTableName)) { throw new IOException("Failed to create a state file for table " + tmpTableName);
@BeforeClass public void setUp() throws Exception { Properties properties = new Properties(); try (FileReader fr = new FileReader("gobblin-test/resource/gobblin.test.properties")) { properties.load(fr); } this.datasetStateStore = new FsStateStore<>( properties.getProperty(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI), properties.getProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY), JobState.DatasetState.class); // clear data that might be there from a prior run this.datasetStateStore.delete(JOB_NAME); this.jobConfig.putAll(properties); this.jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, JOB_NAME); this.jobConfig.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, DummySource.class.getName()); this.jobConfig.setProperty(ConfigurationKeys.WRITER_BUILDER_CLASS, DummyDataWriterBuilder.class.getName()); }
@Override public List<T> getAll(String storeName) throws IOException { List<T> states = Lists.newArrayList(); Path storePath = new Path(this.storeRootDir, storeName); if (!this.fs.exists(storePath)) { return states; } for (FileStatus status : this.fs.listStatus(storePath)) { states.addAll(getAll(storeName, status.getPath().getName())); } return states; }
public FsStateStore(String fsUri, String storeRootDir, Class<T> stateClass) throws IOException { this.conf = getConf(null); this.fs = FileSystem.get(URI.create(fsUri), this.conf); this.useTmpFileForPut = !FS_SCHEMES_NON_ATOMIC.contains(this.fs.getUri().getScheme()); this.storeRootDir = storeRootDir; this.stateClass = stateClass; }
@Override public boolean create(String storeName, String tableName) throws IOException { Path storePath = new Path(this.storeRootDir, storeName); if (!this.fs.exists(storePath) && !create(storeName)) { return false; } Path tablePath = new Path(storePath, tableName); if (this.fs.exists(tablePath)) { throw new IOException(String.format("State file %s already exists for table %s", tablePath, tableName)); } return this.fs.createNewFile(tablePath); }
@BeforeClass public void setUp() throws IOException { this.fsJobStateStore = new FsStateStore<>(ConfigurationKeys.LOCAL_FS_URI, FsDatasetStateStoreTest.class.getSimpleName(), JobState.class); this.fsDatasetStateStore = new FsDatasetStateStore(ConfigurationKeys.LOCAL_FS_URI, FsDatasetStateStoreTest.class.getSimpleName()); // clear data that may have been left behind by a prior test run this.fsDatasetStateStore.delete(TEST_JOB_NAME); }