@Override public void run() { long startTime = System.currentTimeMillis(); onStart(startTime); try { this.underlyingTask.run(); successfulTaskCount.mark();; } catch (Exception e) { failedTaskCount.mark(); throw e; } finally { runningTaskCount.dec(); } }
private ArrayList<ArrayList<Object>> runTaskAndGetResults(TaskState taskState, int numRecords, int numForks, ForkOperator mockForkOperator) throws Exception { ArrayList<ArrayList<Object>> recordCollectors = new ArrayList<>(numForks); for (int i=0; i < numForks; ++i) { recordCollectors.add(new ArrayList<Object>()); } TaskContext mockTaskContext = getMockTaskContext(taskState, new StringExtractor(numRecords), recordCollectors, mockForkOperator); // Create a mock TaskStateTracker TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class); // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread TaskExecutor taskExecutor = new TaskExecutor(new Properties()); // Create the Task Task task = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch>absent()); // Run and commit task.run(); task.commit(); return recordCollectors; }
doNothing().when(task).submitTaskCommittedEvent(); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL);
@Test public void testControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), new BasicTestControlMessage("1"), new RecordEnvelope<>("b"), new BasicTestControlMessage("2")}); MyConverter converter = new MyConverter(); MyDataWriter writer = new MyDataWriter(); Task task = setupTask(extractor, writer, converter); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a", "b")); Assert.assertEquals(converter.messages, Lists.newArrayList(new BasicTestControlMessage("1"), new BasicTestControlMessage("2"))); Assert.assertEquals(writer.records, Lists.newArrayList("a", "b")); Assert.assertEquals(writer.messages, Lists.newArrayList(new BasicTestControlMessage("1"), new BasicTestControlMessage("2"))); }
@Test public void testFlushControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), FlushControlMessage.builder().flushReason("flush1").build(), new RecordEnvelope<>("b"), FlushControlMessage.builder().flushReason("flush2").build()}); MyConverter converter = new MyConverter(); MyFlushDataWriter writer = new MyFlushDataWriter(); Task task = setupTask(extractor, writer, converter); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a", "b")); Assert.assertEquals(converter.messages, Lists.newArrayList( FlushControlMessage.builder().flushReason("flush1").build(), FlushControlMessage.builder().flushReason("flush2").build())); Assert.assertEquals(writer.records, Lists.newArrayList("a", "b")); Assert.assertEquals(writer.flush_messages, Lists.newArrayList("flush called", "flush called")); }
/** * Test the injection of {@link ControlMessage}s * @throws Exception */ @Test public void testInjectedControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("schema:a"), new RecordEnvelope<>("schema:b"), new RecordEnvelope<>("schema1:c"), new RecordEnvelope<>("schema2:d")}); SchemaChangeDetectionInjector injector = new SchemaChangeDetectionInjector(); SchemaAppendConverter converter = new SchemaAppendConverter(); MyDataWriter writer = new MyDataWriterWithSchemaCheck(); Task task = setupTask(extractor, writer, Collections.EMPTY_LIST, Lists.newArrayList(injector, converter)); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a:schema", "b:schema", "c:schema1", "d:schema2")); Assert.assertEquals(converter.messages, Lists.newArrayList(new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("schema1").build()), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("schema2").build()))); Assert.assertEquals(writer.records, Lists.newArrayList("a:schema", "b:schema", "c:schema1", "d:schema2")); Assert.assertEquals(writer.messages, Lists.newArrayList(new MetadataUpdateControlMessage<>( GlobalMetadata.<String>builder().schema("schema1").build()), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("schema2").build()))); }
/** * Test of metadata update control messages that signal the converters to change schemas * @throws Exception */ @Test public void testMetadataUpdateControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema1").build()), new RecordEnvelope<>("b"), new MetadataUpdateControlMessage(GlobalMetadata.<String>builder().schema("Schema2").build())}); SchemaAppendConverter converter = new SchemaAppendConverter(); MyDataWriter writer = new MyDataWriter(); Task task = setupTask(extractor, writer, converter); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a:schema", "b:Schema1")); Assert.assertEquals(converter.messages, Lists.newArrayList(new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema1").build()), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema2").build()))); Assert.assertEquals(writer.records, Lists.newArrayList("a:schema", "b:Schema1")); Assert.assertEquals(writer.messages, Lists.newArrayList(new MetadataUpdateControlMessage<>( GlobalMetadata.<String>builder().schema("Schema1").build()), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema2").build()))); }
/** * Test with the converter configured in the list of {@link RecordStreamProcessor}s. * @throws Exception */ @Test public void testMetadataUpdateWithStreamProcessors() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema1").build()), new RecordEnvelope<>("b"), new MetadataUpdateControlMessage(GlobalMetadata.<String>builder().schema("Schema2").build())}); SchemaAppendConverter converter = new SchemaAppendConverter(); MyDataWriter writer = new MyDataWriter(); Task task = setupTask(extractor, writer, Collections.EMPTY_LIST, Lists.newArrayList(converter)); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a:schema", "b:Schema1")); Assert.assertEquals(converter.messages, Lists.newArrayList(new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema1").build()), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema2").build()))); Assert.assertEquals(writer.records, Lists.newArrayList("a:schema", "b:Schema1")); Assert.assertEquals(writer.messages, Lists.newArrayList(new MetadataUpdateControlMessage<>( GlobalMetadata.<String>builder().schema("Schema1").build()), new MetadataUpdateControlMessage<>(GlobalMetadata.<String>builder().schema("Schema2").build()))); }
@Override public void run() { long startTime = System.currentTimeMillis(); onStart(startTime); try { this.underlyingTask.run(); successfulTaskCount.mark();; } catch (Exception e) { failedTaskCount.mark(); throw e; } finally { runningTaskCount.dec(); } }