if (segmentsAndMetadata.getSegments().isEmpty()) { log.info("Nothing to publish, skipping publish step."); } else { log.info( "Publishing segments with commitMetadata[%s]: [%s]", segmentsAndMetadata.getCommitMetadata(), Joiner.on(", ").join(segmentsAndMetadata.getSegments()) ); final Object metadata = segmentsAndMetadata.getCommitMetadata(); final boolean published = publisher.publishSegments( ImmutableSet.copyOf(segmentsAndMetadata.getSegments()), metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ).isSuccess(); segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly); .getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .equals(Sets.newHashSet(segmentsAndMetadata.getSegments()))) { log.info("Our segments really do exist, awaiting handoff."); } else { log.warn(e, "Failed publish, not removing segments: %s", segmentsAndMetadata.getSegments()); throw Throwables.propagate(e);
/** * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming * ingestion) before being dropped. * * @param segmentsAndMetadata result of pushing or publishing * * @return a future for dropping segments */ ListenableFuture<SegmentsAndMetadata> dropInBackground(SegmentsAndMetadata segmentsAndMetadata) { log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments()); final ListenableFuture<?> dropFuture = Futures.allAsList( segmentsAndMetadata .getSegments() .stream() .map(segment -> appenderator.drop(SegmentIdWithShardSpec.fromDataSegment(segment))) .collect(Collectors.toList()) ); return Futures.transform( dropFuture, (Function<Object, SegmentsAndMetadata>) x -> { final Object metadata = segmentsAndMetadata.getCommitMetadata(); return new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ); } ); }
@Override public void onSuccess(SegmentsAndMetadata result) { // Immediately publish after pushing for (DataSegment pushedSegment : result.getSegments()) { try { segmentPublisher.publishSegment(pushedSegment); } catch (Exception e) { errorHandler.apply(e); } } log.info("Published [%,d] sinks.", segmentsToPush.size()); }
new SegmentsAndMetadata( snapshot .values()
(Function<SegmentsAndMetadata, SegmentsAndMetadata>) segmentsAndMetadata -> { final Set<SegmentIdWithShardSpec> pushedSegments = segmentsAndMetadata.getSegments().stream() .map( SegmentIdWithShardSpec::fromDataSegment) if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) { log.warn( "Removing segments from deep storage because sanity check failed: %s", segmentsAndMetadata.getSegments() ); segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);
return new SegmentsAndMetadata(dataSegments, commitMetadata); }, pushExecutor
).get(); List<?> publishedSegmentIds = Lists.transform(published.getSegments(), DataSegment::getId); log.info( "Published segments %s with metadata[%s].", publishedSegmentIds, Preconditions.checkNotNull(published.getCommitMetadata(), "commitMetadata") ); log.info( "Handoff completed for segments %s with metadata[%s]", Lists.transform(handedOff.getSegments(), DataSegment::getId), Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata") );
final List<SegmentIdWithShardSpec> waitingSegmentIdList = segmentsAndMetadata.getSegments().stream() .map( SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toList()); final Object metadata = Preconditions.checkNotNull(segmentsAndMetadata.getCommitMetadata(), "commitMetadata"); new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), ((AppenderatorDriverMetadata) metadata).getCallerMetadata() log.info("Successfully handed off [%d] segments.", segmentsAndMetadata.getSegments().size()); resultFuture.set( new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), ((AppenderatorDriverMetadata) metadata).getCallerMetadata()
.getSegments() .stream() .collect(Collectors.toMap(SegmentIdWithShardSpec::fromDataSegment, Function.identity()));
return Futures.transform( persistAll(committer), (Function<Object, SegmentsAndMetadata>) commitMetadata -> new SegmentsAndMetadata(segments, commitMetadata) ); } else {
log.info( "Handoff completed for segments %s with metadata[%s].", Lists.transform(handedOff.getSegments(), DataSegment::getId), Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata") );
/** * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming * ingestion) before being dropped. * * @param segmentsAndMetadata result of pushing or publishing * * @return a future for dropping segments */ ListenableFuture<SegmentsAndMetadata> dropInBackground(SegmentsAndMetadata segmentsAndMetadata) { log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments()); final ListenableFuture<?> dropFuture = Futures.allAsList( segmentsAndMetadata .getSegments() .stream() .map(segment -> appenderator.drop(SegmentIdentifier.fromDataSegment(segment))) .collect(Collectors.toList()) ); return Futures.transform( dropFuture, (Function<Object, SegmentsAndMetadata>) x -> { final Object metadata = segmentsAndMetadata.getCommitMetadata(); return new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() ); } ); }
pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments()); pushedSegments.addAll(pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments());
new SegmentsAndMetadata( snapshot .values()
Assert.assertEquals(numSegments, segmentsAndMetadata.getSegments().size()); Assert.assertEquals(numSegments * MAX_ROWS_PER_SEGMENT, segmentsAndMetadata.getCommitMetadata());
final List<SegmentIdentifier> waitingSegmentIdList = segmentsAndMetadata.getSegments().stream() .map(SegmentIdentifier::fromDataSegment) .collect(Collectors.toList()); final Object metadata = Preconditions.checkNotNull(segmentsAndMetadata.getCommitMetadata(), "commitMetadata"); new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), ((AppenderatorDriverMetadata) metadata).getCallerMetadata() log.info("Successfully handed off [%d] segments.", segmentsAndMetadata.getSegments().size()); resultFuture.set( new SegmentsAndMetadata( segmentsAndMetadata.getSegments(), ((AppenderatorDriverMetadata) metadata).getCallerMetadata()
log.info("Pushed segments[%s]", pushed.getSegments()); log.info("Pushed segments[%s]", pushed.getSegments()); buildSegmentsMeters.getThrownAway() ); log.info("Published segments: %s", Lists.transform(published.getSegments(), DataSegment::getId));
return new SegmentsAndMetadata(dataSegments, commitMetadata); }, pushExecutor
@Test public void testSimple() throws Exception { Assert.assertNull(driver.startJob()); for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); } checkSegmentStates(2, SegmentState.APPENDING); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
@Override public void onSuccess(SegmentsAndMetadata result) { // Immediately publish after pushing for (DataSegment pushedSegment : result.getSegments()) { try { segmentPublisher.publishSegment(pushedSegment); } catch (Exception e) { errorHandler.apply(e); } } log.info("Published [%,d] sinks.", segmentsToPush.size()); }