@Override public void onSuccess(SegmentsAndMetadata result) { // Immediately publish after pushing for (DataSegment pushedSegment : result.getSegments()) { try { segmentPublisher.publishSegment(pushedSegment); } catch (Exception e) { errorHandler.apply(e); } } log.info("Published [%,d] sinks.", segmentsToPush.size()); }
(Function<SegmentsAndMetadata, SegmentsAndMetadata>) segmentsAndMetadata -> { final Set<SegmentIdWithShardSpec> pushedSegments = segmentsAndMetadata.getSegments().stream() .map( SegmentIdWithShardSpec::fromDataSegment) if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) { log.warn( "Removing segments from deep storage because sanity check failed: %s", segmentsAndMetadata.getSegments() ); segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);
/** * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming * ingestion) before being dropped. * * @param segmentsAndMetadata result of pushing or publishing * * @return a future for dropping segments */ ListenableFuture<SegmentsAndMetadata> dropInBackground(SegmentsAndMetadata segmentsAndMetadata) { log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments()); final ListenableFuture<?> dropFuture = Futures.allAsList( segmentsAndMetadata .getSegments() .stream() .map(segment -> appenderator.drop(SegmentIdWithShardSpec.fromDataSegment(segment))) .collect(Collectors.toList()) ); return Futures.transform( dropFuture, (Function<Object, SegmentsAndMetadata>) x -> { final Object metadata = segmentsAndMetadata.getCommitMetadata(); return new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ); } ); }
if (segmentsAndMetadata.getSegments().isEmpty()) { log.info("Nothing to publish, skipping publish step."); } else { "Publishing segments with commitMetadata[%s]: [%s]", segmentsAndMetadata.getCommitMetadata(), Joiner.on(", ").join(segmentsAndMetadata.getSegments()) ); final Object metadata = segmentsAndMetadata.getCommitMetadata(); final boolean published = publisher.publishSegments( ImmutableSet.copyOf(segmentsAndMetadata.getSegments()), metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ).isSuccess(); segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly); .getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .equals(Sets.newHashSet(segmentsAndMetadata.getSegments()))) { log.info("Our segments really do exist, awaiting handoff."); } else { log.warn(e, "Failed publish, not removing segments: %s", segmentsAndMetadata.getSegments()); throw Throwables.propagate(e);
.getSegments() .stream() .collect(Collectors.toMap(SegmentIdWithShardSpec::fromDataSegment, Function.identity()));
final List<SegmentIdWithShardSpec> waitingSegmentIdList = segmentsAndMetadata.getSegments().stream() .map( SegmentIdWithShardSpec::fromDataSegment) return Futures.immediateFuture( new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), ((AppenderatorDriverMetadata) metadata).getCallerMetadata()
pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments()); pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments());
log.info( "Handoff completed for segments %s with metadata[%s].", Lists.transform(handedOff.getSegments(), DataSegment::getId), Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata") );
).get(); List<?> publishedSegmentIds = Lists.transform(published.getSegments(), DataSegment::getId); log.info( "Published segments %s with metadata[%s].", log.info( "Handoff completed for segments %s with metadata[%s]", Lists.transform(handedOff.getSegments(), DataSegment::getId), Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata") );
Assert.assertEquals(numSegments, segmentsAndMetadata.getSegments().size()); Assert.assertEquals(numSegments * MAX_ROWS_PER_SEGMENT, segmentsAndMetadata.getCommitMetadata());
log.info("Pushed segments[%s]", pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments()); buildSegmentsMeters.getThrownAway() ); log.info("Published segments: %s", Lists.transform(published.getSegments(), DataSegment::getId));
@Test(timeout = 60_000L) public void testSimple() throws Exception { final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>(); Assert.assertNull(driver.startJob()); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); while (driver.getSegments().containsKey("dummy")) { Thread.sleep(100); } final SegmentsAndMetadata segmentsAndMetadata = driver.registerHandoff(published) .get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), asIdentifiers(segmentsAndMetadata.getSegments()) ); Assert.assertEquals(3, segmentsAndMetadata.getCommitMetadata()); }
@Test public void testSimple() throws Exception { Assert.assertNull(driver.startJob()); for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); } checkSegmentStates(2, SegmentState.APPENDING); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
sorted( Lists.transform( segmentsAndMetadata.getSegments(), new Function<DataSegment, SegmentIdWithShardSpec>() Assert.assertEquals(sorted(tester.getPushedSegments()), sorted(segmentsAndMetadata.getSegments()));
new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), asIdentifiers(segmentsAndMetadata.getSegments()) ); asIdentifiers(segmentsAndMetadata.getSegments()) ); asIdentifiers(segmentsAndMetadata.getSegments()) );
new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), asIdentifiers(handedoffFromSequence0.getSegments()) ); new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), asIdentifiers(handedoffFromSequence1.getSegments()) );
@Test public void testIncrementalPush() throws Exception { Assert.assertNull(driver.startJob()); int i = 0; for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); checkSegmentStates(1, SegmentState.APPENDING); checkSegmentStates(i, SegmentState.PUSHED_AND_DROPPED); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(0, SegmentState.APPENDING); checkSegmentStates(++i, SegmentState.PUSHED_AND_DROPPED); } final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(1, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
@Override public void onSuccess(SegmentsAndMetadata result) { // Immediately publish after pushing for (DataSegment pushedSegment : result.getSegments()) { try { segmentPublisher.publishSegment(pushedSegment); } catch (Exception e) { errorHandler.apply(e); } } log.info("Published [%,d] sinks.", segmentsToPush.size()); }
(Function<SegmentsAndMetadata, SegmentsAndMetadata>) segmentsAndMetadata -> { final Set<SegmentIdentifier> pushedSegments = segmentsAndMetadata.getSegments().stream() .map(SegmentIdentifier::fromDataSegment) .collect(Collectors.toSet()); if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) { log.warn( "Removing segments from deep storage because sanity check failed: %s", segmentsAndMetadata.getSegments() ); segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);
/** * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming * ingestion) before being dropped. * * @param segmentsAndMetadata result of pushing or publishing * * @return a future for dropping segments */ ListenableFuture<SegmentsAndMetadata> dropInBackground(SegmentsAndMetadata segmentsAndMetadata) { log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments()); final ListenableFuture<?> dropFuture = Futures.allAsList( segmentsAndMetadata .getSegments() .stream() .map(segment -> appenderator.drop(SegmentIdentifier.fromDataSegment(segment))) .collect(Collectors.toList()) ); return Futures.transform( dropFuture, (Function<Object, SegmentsAndMetadata>) x -> { final Object metadata = segmentsAndMetadata.getCommitMetadata(); return new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ); } ); }