public boolean isPushRequired(@Nullable Integer maxRowsPerSegment, @Nullable Long maxTotalRows) { boolean overThreshold = false; if (maxRowsPerSegment != null) { overThreshold = getNumRowsInSegment() >= maxRowsPerSegment; } if (maxTotalRows != null) { overThreshold |= getTotalNumRowsInAppenderator() >= maxTotalRows; } return overThreshold; }
public boolean isPushRequired(AppenderatorConfig tuningConfig) { return isPushRequired(tuningConfig.getMaxRowsPerSegment(), tuningConfig.getMaxTotalRows()); }
); if (addResult.isOk()) { if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { segmentsToMoveOut.computeIfAbsent(sequenceName, k -> new HashSet<>()) .add(addResult.getSegmentIdentifier()); isPersistRequired |= addResult.isPersistRequired(); } else { if (addResult.getParseException() != null) { handleParseException(addResult.getParseException(), record); } else { rowIngestionMeters.incrementProcessed();
if (addResult.isOk()) { if (!isGuaranteedRollup && addResult.isPushRequired(maxRowsPerSegment, maxTotalRows)) { if (addResult.getParseException() != null) { handleParseException(addResult.getParseException()); } else { buildSegmentsMeters.incrementProcessed();
if (addResult.isOk()) { (exceedMaxRowsInSegment(targetPartitionSize, addResult.getNumRowsInSegment()) || exceedMaxRowsInAppenderator(maxTotalRows, addResult.getTotalNumRowsInAppenderator()))) { if (addResult.getParseException() != null) { handleParseException(addResult.getParseException()); } else { buildSegmentsMeters.incrementProcessed();
Assert.assertTrue(addResult.isOk()); if (addResult.getNumRowsInSegment() > MAX_ROWS_PER_SEGMENT) { driver.moveSegmentOut("dummy", ImmutableList.of(addResult.getSegmentIdentifier()));
); if (addResult.isOk()) { if (addResult.isPushRequired(tuningConfig) && !sequenceToUse.isCheckpointed()) { sequenceToCheckpoint = sequenceToUse; isPersistRequired |= addResult.isPersistRequired(); } else { if (addResult.getParseException() != null) { handleParseException(addResult.getParseException(), record); } else { rowIngestionMeters.incrementProcessed();
@Test(timeout = 60_000L, expected = TimeoutException.class) public void testHandoffTimeout() throws Exception { final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.disableHandoff(); Assert.assertNull(driver.startJob()); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); while (driver.getSegments().containsKey("dummy")) { Thread.sleep(100); } driver.registerHandoff(published).get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); }
final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); if (addResult.isOk()) { if (exceedMaxRowsInSegment(targetPartitionSize, addResult.getNumRowsInSegment()) || exceedMaxRowsInAppenderator(maxTotalRows, addResult.getTotalNumRowsInAppenderator())) {
final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); if (addResult.isOk()) { if (addResult.isPushRequired(maxRowsPerSegment, maxTotalRows)) {
AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName, committerSupplier); if (addResult.isOk()) { if (addResult.isPushRequired(tuningConfig)) { publishSegments(driver, publisher, committerSupplier, sequenceName); sequenceNumber++; if (addResult.getParseException() != null) { handleParseException(addResult.getParseException()); } else { rowIngestionMeters.incrementProcessed();
@Test(timeout = 60_000L) public void testSimple() throws Exception { final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>(); Assert.assertNull(driver.startJob()); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); while (driver.getSegments().containsKey("dummy")) { Thread.sleep(100); } final SegmentsAndMetadata segmentsAndMetadata = driver.registerHandoff(published) .get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), asIdentifiers(segmentsAndMetadata.getSegments()) ); Assert.assertEquals(3, segmentsAndMetadata.getCommitMetadata()); }
AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName, committerSupplier); if (addResult.isOk()) { if (addResult.isPushRequired(tuningConfig)) { publishSegments(driver, publisher, committerSupplier, sequenceName); sequenceNumber++; if (addResult.getParseException() != null) { handleParseException(addResult.getParseException()); } else { rowIngestionMeters.incrementProcessed();
@Test public void testSimple() throws Exception { Assert.assertNull(driver.startJob()); for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); } checkSegmentStates(2, SegmentState.APPENDING); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
public boolean isPushRequired(AppenderatorConfig tuningConfig) { boolean overThreshold = getNumRowsInSegment() >= tuningConfig.getMaxRowsPerSegment(); Long maxTotal = tuningConfig.getMaxTotalRows(); if (maxTotal != null) { overThreshold |= getTotalNumRowsInAppenderator() >= maxTotal; } return overThreshold; }
Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());
Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());
Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());
Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());