@Override public Void call() throws Exception { stateStore.put(storeName, fileName, workUnit); return null; } }, "Serialize state to store " + storeName + " file " + fileName);
@Override public void commitWatermarks(Iterable<CheckpointableWatermark> watermarks) throws IOException { for (CheckpointableWatermark watermark: watermarks) { String tableName = watermark.getSource(); _stateStore.put(_storeName, tableName, new CheckpointableWatermarkState(watermark, GSON)); } }
public void setPlanningJobId (String jobName, String planningJobId) throws IOException { State state = getOrCreate(distributedStateStoreName, jobName); state.setId(jobName); state.setProp(GobblinClusterConfigurationKeys.PLANNING_ID_KEY, planningJobId); // fs state store use hdfs rename, which assumes the target file doesn't exist. if (stateStore instanceof FsStateStore) { this.deleteMapping(jobName); } this.stateStore.put(distributedStateStoreName, jobName, state); }
public void setActualJobId (String jobName, String planningJobId, String actualJobId) throws IOException { State state = getOrCreate(distributedStateStoreName, jobName); state.setId(jobName); state.setProp(GobblinClusterConfigurationKeys.PLANNING_ID_KEY, planningJobId); state.setProp(ConfigurationKeys.JOB_ID_KEY, actualJobId); // fs state store use hdfs rename, which assumes the target file doesn't exist. if (stateStore instanceof FsStateStore) { this.deleteMapping(jobName); } this.stateStore.put(distributedStateStoreName, jobName, state); }
for (Task task : tasks) { log.info("Writing task state for task " + task.getTaskId()); taskStateStore.put(task.getJobId(), task.getTaskId() + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, task.getTaskState()); taskStateStore.put(task.getJobId(), task.getTaskId() + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX, task.getTaskState());
this.stateStores.getJobStateStore().put(jobStateFilePath.getParent().getName(), jobStateFilePath.getName(), this.jobContext.getJobState()); } else {
dbJobStateStore.put(TEST_JOB_NAME, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); dbJobStateStore.put(TEST_JOB_NAME_LOWER, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); dbJobStateStore.put(TEST_JOB_NAME2, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState);
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } zkJobStateStore.put(TEST_JOB_NAME, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); // second job name for testing getting store names in a later test case jobState.setJobName(TEST_JOB_NAME2); zkJobStateStore.put(TEST_JOB_NAME2, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } this.fsJobStateStore.put(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
@Override public Void call() throws Exception { stateStore.put(storeName, fileName, workUnit); return null; } }, "Serialize state to store " + storeName + " file " + fileName);
@Override public void commitWatermarks(Iterable<CheckpointableWatermark> watermarks) throws IOException { for (CheckpointableWatermark watermark: watermarks) { String tableName = watermark.getSource(); _stateStore.put(_storeName, tableName, new CheckpointableWatermarkState(watermark, GSON)); } }
@VisibleForTesting protected void setResultToUserContent(Map<String, String> keyValues) throws IOException { WorkUnitState wus = new WorkUnitState(); wus.setProp(ConfigurationKeys.JOB_ID_KEY, this.planningJobId); wus.setProp(ConfigurationKeys.TASK_ID_KEY, this.planningJobId); wus.setProp(ConfigurationKeys.TASK_KEY_KEY, this.planningJobId); keyValues.forEach((key, value)->wus.setProp(key, value)); TaskState taskState = new TaskState(wus); this.stateStores.getTaskStateStore().put(this.planningJobId, this.planningJobId, taskState); }
for (Task task : tasks) { log.info("Writing task state for task " + task.getTaskId()); taskStateStore.put(task.getJobId(), task.getTaskId() + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, task.getTaskState()); taskStateStore.put(task.getJobId(), task.getTaskId() + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX, task.getTaskState());
this.stateStores.getJobStateStore().put(jobStateFilePath.getParent().getName(), jobStateFilePath.getName(), this.jobContext.getJobState()); } else {