Assert.assertEquals(0, appenderator.getTotalRowCount()); appenderator.startJob(); Assert.assertEquals(0, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.persistAll(committerSupplier.get()).get(); Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.drop(IDENTIFIERS.get(0)).get(); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.drop(IDENTIFIERS.get(1)).get(); Assert.assertEquals(0, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "bar", 1), committerSupplier); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "baz", 1), committerSupplier); Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "qux", 1), committerSupplier); Assert.assertEquals(3, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "bob", 1), committerSupplier); Assert.assertEquals(4, appenderator.getTotalRowCount()); appenderator.persistAll(committerSupplier.get()).get(); Assert.assertEquals(4, appenderator.getTotalRowCount()); appenderator.drop(IDENTIFIERS.get(2)).get(); Assert.assertEquals(0, appenderator.getTotalRowCount());
/** * Clears out all our state and also calls {@link Appenderator#clear()} on the underlying Appenderator. */ public void clear() throws InterruptedException { synchronized (segments) { segments.clear(); } appenderator.clear(); }
/** * Create a driver. * * @param appenderator appenderator * @param segmentAllocator segment allocator * @param handoffNotifierFactory handoff notifier factory * @param usedSegmentChecker used segment checker * @param objectMapper object mapper, used for serde of commit metadata * @param metrics Firedepartment metrics */ public StreamAppenderatorDriver( Appenderator appenderator, SegmentAllocator segmentAllocator, SegmentHandoffNotifierFactory handoffNotifierFactory, UsedSegmentChecker usedSegmentChecker, DataSegmentKiller dataSegmentKiller, ObjectMapper objectMapper, FireDepartmentMetrics metrics ) { super(appenderator, segmentAllocator, usedSegmentChecker, dataSegmentKiller); this.handoffNotifier = Preconditions.checkNotNull(handoffNotifierFactory, "handoffNotifierFactory") .createSegmentHandoffNotifier(appenderator.getDataSource()); this.metrics = Preconditions.checkNotNull(metrics, "metrics"); this.objectMapper = Preconditions.checkNotNull(objectMapper, "objectMapper"); }
List<SegmentIdWithShardSpec> pending = appenderator.getSegments(); if (pending.isEmpty()) { log.info("No segments to hand off."); log.info("Waiting to hand off: %s", Joiner.on(", ").join(pending)); handoffCondition.wait(); pending = appenderator.getSegments(); handoffNotifier.close(); shutdownExecutors(); appenderator.close();
final Appenderator.AppenderatorAddResult result = appenderator.add( identifier, row, identifier, result.getNumRowsInSegment(), appenderator.getTotalRowCount(), result.isPersistRequired(), result.getParseException()
Assert.assertEquals(null, appenderator.startJob()); Assert.assertEquals(AppenderatorTester.DATASOURCE, appenderator.getDataSource()); Assert.assertEquals( 1, appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals( 2, appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals( 1, appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals(IDENTIFIERS.subList(0, 2), sorted(appenderator.getSegments())); Assert.assertEquals(2, appenderator.getRowCount(IDENTIFIERS.get(0))); Assert.assertEquals(1, appenderator.getRowCount(IDENTIFIERS.get(1))); thrown = false; try { appenderator.getRowCount(IDENTIFIERS.get(2));
appenderator.startJob(); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 3), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "qux", 4), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 5), committerSupplier); appenderator.close(); )) { final Appenderator appenderator2 = tester2.getAppenderator(); Assert.assertEquals(ImmutableMap.of("eventCount", 4), appenderator2.startJob()); Assert.assertEquals(ImmutableList.of(IDENTIFIERS.get(0)), appenderator2.getSegments()); Assert.assertEquals(4, appenderator2.getRowCount(IDENTIFIERS.get(0)));
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier, false); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 1), committerSupplier, false); Assert.assertEquals(3, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 1), committerSupplier, false); Assert.assertEquals(4, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier, false); Assert.assertEquals(5, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get()); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close();
}; appenderator.startJob(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals( 276 + 2 * nullHandlingOverhead, ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory() ); appenderator.close(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory());
new FireDepartmentMetrics() )) { Assert.assertEquals("dataSourceName", appenderator.getDataSource()); Assert.assertEquals(null, appenderator.startJob()); SegmentIdWithShardSpec identifier = new SegmentIdWithShardSpec( "dataSourceName", ); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(identifier, AppenderatorTest.IR("2000", "bar", 1), Suppliers.ofInstance(Committers.nil())); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(identifier, AppenderatorTest.IR("2000", "baz", 1), Suppliers.ofInstance(Committers.nil())); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory());
Assert.assertEquals(AppenderatorTester.DATASOURCE, appenderator.getDataSource()); Assert.assertEquals(3, appenderator.getRowCount(si)); appenderator.clear(); Assert.assertTrue(appenderator.getSegments().isEmpty());
long minTimestamp = minTimestampAsDate.getMillis(); final List<SegmentIdWithShardSpec> appenderatorSegments = appenderator.getSegments(); final List<SegmentIdWithShardSpec> segmentsToPush = new ArrayList<>(); appenderator.push(segmentsToPush, Committers.nil(), false), new FutureCallback<SegmentsAndMetadata>()
final Appenderator appenderator = tester.getAppenderator(); appenderator.startJob(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 2), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(1), IR("2000", "foo", 4), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001", "foo", 8), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
/** * Persist all data indexed through this driver so far. Returns a future of persisted commitMetadata. * <p> * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param committer committer representing all data that has been added so far * * @return future containing commitMetadata persisted */ public ListenableFuture<Object> persistAsync(final Committer committer) { return appenderator.persistAll(wrapCommitter(committer)); }
/** * This method always returns null because batch ingestion doesn't support restoring tasks on failures. * * @return always null */ @Override @Nullable public Object startJob() { final Object metadata = appenderator.startJob(); if (metadata != null) { throw new ISE("Metadata should be null because BatchAppenderatorDriver never persists it"); } return null; }
public void dropSegment(final SegmentIdWithShardSpec identifier) { log.info("Dropping segment: %s", identifier); segments.remove(identifier.getInterval().getStartMillis()); Futures.addCallback( appenderator.drop(identifier), new FutureCallback<Object>() { @Override public void onSuccess(Object result) { log.info("Dropped segment: %s", identifier); } @Override public void onFailure(Throwable e) { // TODO: Retry? log.warn(e, "Failed to drop segment: %s", identifier); } } ); }
@Override public void close() throws Exception { appenderator.close(); queryExecutor.shutdownNow(); emitter.close(); FileUtils.deleteDirectory(tuningConfig.getBasePersistDirectory()); } }
/** * Same as {@link #add(SegmentIdWithShardSpec, InputRow, Supplier, boolean)}, with allowIncrementalPersists set to true */ default AppenderatorAddResult add(SegmentIdWithShardSpec identifier, InputRow row, Supplier<Committer> committerSupplier) throws IndexSizeExceededException, SegmentNotWritableException { return add(identifier, row, committerSupplier, true); }
appenderator.push(segmentIdentifiers, wrappedCommitter, useUniquePath), (Function<SegmentsAndMetadata, SegmentsAndMetadata>) segmentsAndMetadata -> {