/** * Same as {@link #add(SegmentIdWithShardSpec, InputRow, Supplier, boolean)}, with allowIncrementalPersists set to true */ default AppenderatorAddResult add(SegmentIdWithShardSpec identifier, InputRow row, Supplier<Committer> committerSupplier) throws IndexSizeExceededException, SegmentNotWritableException { return add(identifier, row, committerSupplier, true); }
@Override public IncrementalIndexAddResult add(InputRow row, Supplier<Committer> committerSupplier) throws IndexSizeExceededException { final SegmentIdWithShardSpec identifier = getSegmentIdentifier(row.getTimestampFromEpoch()); if (identifier == null) { return Plumber.THROWAWAY; } try { final Appenderator.AppenderatorAddResult addResult = appenderator.add(identifier, row, committerSupplier); lastCommitterSupplier = committerSupplier; return new IncrementalIndexAddResult(addResult.getNumRowsInSegment(), 0, addResult.getParseException()); } catch (SegmentNotWritableException e) { // Segment already started handoff return Plumber.NOT_WRITABLE; } }
final Appenderator.AppenderatorAddResult result = appenderator.add( identifier, row,
appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 3), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "qux", 4), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 5), committerSupplier); appenderator.close();
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier, false); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 1), committerSupplier, false); Assert.assertEquals(3, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 1), committerSupplier, false); Assert.assertEquals(4, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier, false); Assert.assertEquals(5, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get());
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 1), committerSupplier); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 1), committerSupplier); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get());
appenderator.startJob(); Assert.assertEquals(0, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, appenderator.getTotalRowCount()); Assert.assertEquals(0, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "bar", 1), committerSupplier); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "baz", 1), committerSupplier); Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "qux", 1), committerSupplier); Assert.assertEquals(3, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "bob", 1), committerSupplier); Assert.assertEquals(4, appenderator.getTotalRowCount());
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); ); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals( 276 + 2 * nullHandlingOverhead,
Assert.assertEquals( 1, appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals( 2, appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals( 1, appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier) .getNumRowsInSegment() );
appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 2), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(1), IR("2000", "foo", 4), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001", "foo", 8), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 2), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(1), IR("2000", "foo", 4), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001", "foo", 8), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(identifier, AppenderatorTest.IR("2000", "bar", 1), Suppliers.ofInstance(Committers.nil())); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(identifier, AppenderatorTest.IR("2000", "baz", 1), Suppliers.ofInstance(Committers.nil())); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close();
/** * Same as {@link #add(SegmentIdentifier, InputRow, Supplier, boolean)}, with allowIncrementalPersists set to true */ default AppenderatorAddResult add(SegmentIdentifier identifier, InputRow row, Supplier<Committer> committerSupplier) throws IndexSizeExceededException, SegmentNotWritableException { return add(identifier, row, committerSupplier, true); }
@Override public IncrementalIndexAddResult add(InputRow row, Supplier<Committer> committerSupplier) throws IndexSizeExceededException { final SegmentIdentifier identifier = getSegmentIdentifier(row.getTimestampFromEpoch()); if (identifier == null) { return Plumber.THROWAWAY; } try { final Appenderator.AppenderatorAddResult addResult = appenderator.add(identifier, row, committerSupplier); lastCommitterSupplier = committerSupplier; return new IncrementalIndexAddResult(addResult.getNumRowsInSegment(), 0, addResult.getParseException()); } catch (SegmentNotWritableException e) { // Segment already started handoff return Plumber.NOT_WRITABLE; } }
final Appenderator.AppenderatorAddResult result = appenderator.add( identifier, row,