@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { // Add all job configuration properties so they are picked up by Hadoop Configuration conf = new Configuration(); for (Map.Entry<String, ConfigValue> entry : config.entrySet()) { conf.set(entry.getKey(), entry.getValue().unwrapped().toString()); } try { String stateStoreFsUri = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem stateStoreFs = FileSystem.get(URI.create(stateStoreFsUri), conf); String stateStoreRootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); return new FsStateStore(stateStoreFs, stateStoreRootDir, stateClass); } catch (IOException e) { throw new RuntimeException("Failed to create FsStateStore with factory", e); } } }
@BeforeClass public void setUp() throws Exception { this.localFs = FileSystem.getLocal(new Configuration()); this.localFs.mkdirs(this.outputTaskStateDir); this.taskStateStore = new FsStateStore<>(this.localFs, this.outputTaskStateDir.toUri().getPath(), TaskState.class); this.taskStateCollectorService = new TaskStateCollectorService(new Properties(), this.jobState, this.eventBus, this.taskStateStore, new Path(this.outputTaskStateDir, JOB_ID)); this.eventBus.register(this); }
taskStateStore = new FsStateStore<>(this.fs, jobOutputPath.toString(), TaskState.class);
@BeforeClass public void setUp() throws Exception { Properties properties = new Properties(); try (FileReader fr = new FileReader("gobblin-test/resource/gobblin.test.properties")) { properties.load(fr); } this.datasetStateStore = new FsStateStore<>( properties.getProperty(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI), properties.getProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY), JobState.DatasetState.class); // clear data that might be there from a prior run this.datasetStateStore.delete(JOB_NAME); this.jobConfig.putAll(properties); this.jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, JOB_NAME); this.jobConfig.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, DummySource.class.getName()); this.jobConfig.setProperty(ConfigurationKeys.WRITER_BUILDER_CLASS, DummyDataWriterBuilder.class.getName()); }
@BeforeClass public void setUp() throws IOException { this.fsJobStateStore = new FsStateStore<>(ConfigurationKeys.LOCAL_FS_URI, FsDatasetStateStoreTest.class.getSimpleName(), JobState.class); this.fsDatasetStateStore = new FsDatasetStateStore(ConfigurationKeys.LOCAL_FS_URI, FsDatasetStateStoreTest.class.getSimpleName()); // clear data that may have been left behind by a prior test run this.fsDatasetStateStore.delete(TEST_JOB_NAME); }
@BeforeClass public void startUp() throws Exception { testMetastoreDatabase = TestMetastoreDatabaseFactory.get(); this.launcherProps = new Properties(); this.launcherProps.load(new FileReader("gobblin-test/resource/gobblin.test.properties")); this.launcherProps.setProperty(ConfigurationKeys.JOB_HISTORY_STORE_ENABLED_KEY, "true"); this.launcherProps.setProperty(ConfigurationKeys.METRICS_ENABLED_KEY, "true"); this.launcherProps.setProperty(ConfigurationKeys.METRICS_REPORTING_FILE_ENABLED_KEY, "false"); this.launcherProps.setProperty(ConfigurationKeys.JOB_HISTORY_STORE_URL_KEY, testMetastoreDatabase.getJdbcUrl()); StateStore<JobState.DatasetState> datasetStateStore = new FsStateStore<>(this.launcherProps.getProperty(ConfigurationKeys.STATE_STORE_FS_URI_KEY), this.launcherProps.getProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY), JobState.DatasetState.class); this.jobLauncherTestHelper = new JobLauncherTestHelper(this.launcherProps, datasetStateStore); }
@BeforeClass public void startUp() throws Exception { this.testLogger.info("startUp: in"); testMetastoreDatabase = TestMetastoreDatabaseFactory.get(); this.launcherProps = new Properties(); try (InputStream propsReader = getClass().getClassLoader().getResourceAsStream("gobblin.mr-test.properties")) { this.launcherProps.load(propsReader); } this.launcherProps.setProperty(ConfigurationKeys.JOB_HISTORY_STORE_ENABLED_KEY, "true"); this.launcherProps.setProperty(ConfigurationKeys.METRICS_ENABLED_KEY, "true"); this.launcherProps.setProperty(ConfigurationKeys.METRICS_REPORTING_FILE_ENABLED_KEY, "false"); this.launcherProps.setProperty(ConfigurationKeys.JOB_HISTORY_STORE_URL_KEY, testMetastoreDatabase.getJdbcUrl()); StateStore<JobState.DatasetState> datasetStateStore = new FsStateStore<>(this.launcherProps.getProperty(ConfigurationKeys.STATE_STORE_FS_URI_KEY), this.launcherProps.getProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY), JobState.DatasetState.class); this.jobLauncherTestHelper = new JobLauncherTestHelper(this.launcherProps, datasetStateStore); // Other tests may not clean up properly, clean up outputDir and stagingDir or some of these tests might fail. String outputDir = this.launcherProps.getProperty(ConfigurationKeys.WRITER_OUTPUT_DIR); String stagingDir = this.launcherProps.getProperty(ConfigurationKeys.WRITER_STAGING_DIR); FileUtils.deleteDirectory(new File(outputDir)); FileUtils.deleteDirectory(new File(stagingDir)); this.testLogger.info("startUp: out"); }
this.fs = FileSystem.get(context.getConfiguration()); this.taskStateStore = new FsStateStore<>(this.fs, FileOutputFormat.getOutputPath(context).toUri().getPath(), TaskState.class);
FsStateStore<WorkUnit> wuStateStore = new FsStateStore<>(this.localFs, workUnitDirPath.toString(), WorkUnit.class); Path workUnitFilePath = new Path(new Path(workUnitDirPath, TestHelper.TEST_JOB_ID), TestHelper.TEST_JOB_NAME + ".wu");
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { // Add all job configuration properties so they are picked up by Hadoop Configuration conf = new Configuration(); for (Map.Entry<String, ConfigValue> entry : config.entrySet()) { conf.set(entry.getKey(), entry.getValue().unwrapped().toString()); } try { String stateStoreFsUri = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem stateStoreFs = FileSystem.get(URI.create(stateStoreFsUri), conf); String stateStoreRootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); return new FsStateStore(stateStoreFs, stateStoreRootDir, stateClass); } catch (IOException e) { throw new RuntimeException("Failed to create FsStateStore with factory", e); } } }
taskStateStore = new FsStateStore<>(this.fs, jobOutputPath.toString(), TaskState.class);
this.fs = FileSystem.get(context.getConfiguration()); this.taskStateStore = new FsStateStore<>(this.fs, FileOutputFormat.getOutputPath(context).toUri().getPath(), TaskState.class);