/** * Get a new {@link GobblinMetrics} instance for a given job. * * @param jobName job name * @param jobId job ID * @return a new {@link GobblinMetrics} instance for the given job */ public static JobMetrics get(String jobName, String jobId) { return get(new JobState(jobName, jobId)); }
@BeforeClass public void setUp() { this.jobState = new JobState("TestJob", "TestJob-1"); }
private JobState getJobState() throws java.io.IOException { JobState jobState; // read the state from the state store if present, otherwise deserialize directly from the file if (_stateStores.haveJobStateStore()) { jobState = _stateStores.getJobStateStore().get(_jobStateFilePath.getParent().getName(), _jobStateFilePath.getName(), _jobStateFilePath.getParent().getName()); } else { jobState = new JobState(); SerializationUtils.deserializeState(_fs, _jobStateFilePath, jobState); } return jobState; }
deserializeConf)) { Object writable = reader.getValueClass() == JobState.class ? new JobState() : new JobState.DatasetState();
@Test(dependsOnMethods = {"testSetAndGet"}) public void testSerDe() throws IOException { Closer closer = Closer.create(); try { ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream()); DataOutputStream dos = closer.register(new DataOutputStream(baos)); this.jobState.write(dos); ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray()))); DataInputStream dis = closer.register((new DataInputStream(bais))); JobState newJobState = new JobState(); newJobState.readFields(dis); doAsserts(newJobState, true, false); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
String jobId = "job_123"; JobState jobState = new JobState(jobName, jobId); JobMetrics jobMetrics = JobMetrics.get(jobState);
@Test public void testErrorMessage() throws Exception { JobContext jobContext = Mockito.mock(JobContext.class); JobState jobState = new JobState("jobName", "1234"); TaskState taskState1 = new TaskState(); TaskState taskState2 = new TaskState();
jobPropsState.addAll(jobProps); this.jobState = new JobState(jobPropsState, this.jobName, this.jobId); this.jobState.setBroker(this.jobBroker); this.jobState.setWorkUnitAndDatasetStateFunctional(new CombinedWorkUnitAndDatasetStateGenerator(this.datasetStateStore, this.jobName));
@Test public void testProcessedCount() throws Exception { JobContext jobContext = Mockito.mock(JobContext.class); JobState jobState = new JobState("jobName", "1234"); TaskState taskState1 = new TaskState(); TaskState taskState2 = new TaskState(); taskState1.setTaskId("1"); taskState1.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "1"); taskState2.setTaskId("2"); taskState2.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "22"); jobState.addTaskState(taskState1); jobState.addTaskState(taskState2); Mockito.when(jobContext.getJobState()).thenReturn(jobState); ClusterEventMetadataGenerator metadataGenerator = new ClusterEventMetadataGenerator(); Map<String, String> metadata; // processed count is not in job cancel event metadata = metadataGenerator.getMetadata(jobContext, EventName.JOB_CANCEL); Assert.assertEquals(metadata.get("processedCount"), null); // processed count is in job complete event metadata = metadataGenerator.getMetadata(jobContext, EventName.getEnumFromEventId("JobCompleteTimer")); Assert.assertEquals(metadata.get("processedCount"), "23"); }
@Test public void testTaskGetMetrics() { String jobId = "job_456"; String taskId = "task_456"; String jobName = "jobName"; JobState jobState = new JobState(jobName, jobId); JobMetrics jobMetrics = JobMetrics.get(jobState); State props = new State(); props.setProp(ConfigurationKeys.JOB_ID_KEY, jobId); props.setProp(ConfigurationKeys.TASK_ID_KEY, taskId); SourceState sourceState = new SourceState(props, new ArrayList<WorkUnitState>()); WorkUnit workUnit = new WorkUnit(sourceState, null); WorkUnitState workUnitState = new WorkUnitState(workUnit); TaskState taskState = new TaskState(workUnitState); TaskMetrics taskMetrics = new TaskMetrics(taskState); Assert.assertNotNull(taskMetrics.getMetricContext()); Assert.assertTrue(taskMetrics.getMetricContext().getParent().isPresent()); Assert.assertEquals(taskMetrics.getMetricContext().getParent().get(), jobMetrics.getMetricContext()); }
private TaskContext getTaskContextForRun(WorkUnit workUnit) { workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, "job123"); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, "task123"); workUnit.setProp(HiveConverterUtils.HIVE_DATASET_DESTINATION_SKIP_SETGROUP, Boolean.toString(true)); HiveTask.disableHiveWatermarker(workUnit); JobState jobState = new JobState("job", "job123"); return new TaskContext(new WorkUnitState(workUnit, jobState)); }
JobState jobState = new JobState(); jobState.setJobName(TestHelper.TEST_JOB_NAME); jobState.setJobId(TestHelper.TEST_JOB_ID);
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } this.fsJobStateStore.put(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
JobState jobState = new JobState(TEST_JOB_NAME1, getJobId(TEST_JOB_ID, 1)); jobState.setId(getJobId(TEST_JOB_ID, 1)); jobState.setProp("foo", "bar");
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } zkJobStateStore.put(TEST_JOB_NAME, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); // second job name for testing getting store names in a later test case jobState.setJobName(TEST_JOB_NAME2); zkJobStateStore.put(TEST_JOB_NAME2, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar");
/** * Get a new {@link GobblinMetrics} instance for a given job. * * @param jobName job name * @param jobId job ID * @return a new {@link GobblinMetrics} instance for the given job */ public static JobMetrics get(String jobName, String jobId) { return get(new JobState(jobName, jobId)); }
private JobState getJobState() throws java.io.IOException { JobState jobState; // read the state from the state store if present, otherwise deserialize directly from the file if (_stateStores.haveJobStateStore()) { jobState = _stateStores.getJobStateStore().get(_jobStateFilePath.getParent().getName(), _jobStateFilePath.getName(), _jobStateFilePath.getParent().getName()); } else { jobState = new JobState(); SerializationUtils.deserializeState(_fs, _jobStateFilePath, jobState); } return jobState; }
deserializeConf)) { Object writable = reader.getValueClass() == JobState.class ? new JobState() : new JobState.DatasetState();