private static BatchAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, final SegmentAllocator segmentAllocator ) { return new BatchAppenderatorDriver( appenderator, segmentAllocator, new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), toolbox.getDataSegmentKiller() ); }
/** * Add a row. Must not be called concurrently from multiple threads. * * @param row the row to add * @param sequenceName sequenceName for this row's segment * * @return {@link AppenderatorDriverAddResult} * * @throws IOException if there is an I/O error while allocating or writing to a segment */ public AppenderatorDriverAddResult add( InputRow row, String sequenceName ) throws IOException { return append(row, sequenceName, null, false, true); }
) throws InterruptedException, ExecutionException, TimeoutException final Map<SegmentIdWithShardSpec, SegmentWithState> requestedSegmentIdsForSequences = getAppendingSegments(sequenceNames) .collect(Collectors.toMap(SegmentWithState::getSegmentIdentifier, Function.identity())); pushInBackground(null, requestedSegmentIdsForSequences.keySet(), false), this::dropInBackground );
@Test public void testSimple() throws Exception { Assert.assertNull(driver.startJob()); for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); } checkSegmentStates(2, SegmentState.APPENDING); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob(); final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments()); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments());
private void checkSegmentStates(int expectedNumSegmentsInState, SegmentState expectedState) { final SegmentsForSequence segmentsForSequence = driver.getSegments().get("dummy"); Assert.assertNotNull(segmentsForSequence); final List<SegmentWithState> segmentWithStates = segmentsForSequence .allSegmentStateStream() .filter(segmentWithState -> segmentWithState.getState() == expectedState) .collect(Collectors.toList()); Assert.assertEquals(expectedNumSegmentsInState, segmentWithStates.size()); }
@Test public void testIncrementalPush() throws Exception { Assert.assertNull(driver.startJob()); int i = 0; for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); checkSegmentStates(1, SegmentState.APPENDING); checkSegmentStates(i, SegmentState.PUSHED_AND_DROPPED); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(0, SegmentState.APPENDING); checkSegmentStates(++i, SegmentState.PUSHED_AND_DROPPED); } final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(1, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob(); final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments()); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments());
) throws InterruptedException, ExecutionException, TimeoutException final Map<SegmentIdentifier, SegmentWithState> requestedSegmentIdsForSequences = getAppendingSegments(sequenceNames) .collect(Collectors.toMap(SegmentWithState::getSegmentIdentifier, Function.identity())); pushInBackground(null, requestedSegmentIdsForSequences.keySet(), false), this::dropInBackground );
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob(); final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); log.info("Pushed segments[%s]", pushed.getSegments()); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); log.info("Pushed segments[%s]", pushed.getSegments()); driver.publishAll(publisher), pushTimeout );
private static BatchAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, final SegmentAllocator segmentAllocator ) { return new BatchAppenderatorDriver( appenderator, segmentAllocator, new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), toolbox.getDataSegmentKiller() ); } }
/** * Add a row. Must not be called concurrently from multiple threads. * * @param row the row to add * @param sequenceName sequenceName for this row's segment * * @return {@link AppenderatorDriverAddResult} * * @throws IOException if there is an I/O error while allocating or writing to a segment */ public AppenderatorDriverAddResult add( InputRow row, String sequenceName ) throws IOException { return append(row, sequenceName, null, false, true); }
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob(); final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); log.info("Pushed segments[%s]", pushed.getSegments()); final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); log.info("Pushed segments[%s]", pushed.getSegments()); driver.publishAll(publisher), pushTimeout );
@Before public void setup() { appenderatorTester = new AppenderatorTester(MAX_ROWS_IN_MEMORY); allocator = new TestSegmentAllocator(DATA_SOURCE, Granularities.HOUR); dataSegmentKiller = createStrictMock(DataSegmentKiller.class); driver = new BatchAppenderatorDriver( appenderatorTester.getAppenderator(), allocator, new TestUsedSegmentChecker(appenderatorTester), dataSegmentKiller ); EasyMock.replay(dataSegmentKiller); }
private static BatchAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, final SegmentAllocator segmentAllocator ) { return new BatchAppenderatorDriver( appenderator, segmentAllocator, new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), toolbox.getDataSegmentKiller() ); }
private static BatchAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, final SegmentAllocator segmentAllocator ) { return new BatchAppenderatorDriver( appenderator, segmentAllocator, new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), toolbox.getDataSegmentKiller() ); } }