private Sink getSink(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final VersioningPolicy versioningPolicy = config.getVersioningPolicy(); DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp)); final long truncatedTime = truncatedDateTime.getMillis(); Sink retVal = sinks.get(truncatedTime); if (retVal == null) { final Interval sinkInterval = new Interval( truncatedDateTime, segmentGranularity.increment(truncatedDateTime) ); retVal = new Sink( sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getMaxRowsInMemory(), TuningConfigs.getMaxBytesInMemoryOrDefault(config.getMaxBytesInMemory()), config.isReportParseExceptions(), config.getDedupColumn() ); addSink(retVal); } return retVal; }
public RealtimeTuningConfig withBasePersistDirectory(File dir) { return new RealtimeTuningConfig( maxRowsInMemory, maxBytesInMemory, intermediatePersistPeriod, windowPeriod, dir, versioningPolicy, rejectionPolicyFactory, maxPendingPersists, shardSpec, indexSpec, true, persistThreadPriority, mergeThreadPriority, reportParseExceptions, handoffConditionTimeout, alertTimeout, segmentWriteOutMediumFactory, dedupColumn ); } }
@Test public void testSerdeWithDefaults() throws Exception { String jsonStr = "{\"type\":\"realtime\"}"; ObjectMapper mapper = TestHelper.makeJsonMapper(); RealtimeTuningConfig config = (RealtimeTuningConfig) mapper.readValue( mapper.writeValueAsString( mapper.readValue( jsonStr, TuningConfig.class ) ), TuningConfig.class ); Assert.assertNotNull(config.getBasePersistDirectory()); Assert.assertEquals(0, config.getHandoffConditionTimeout()); Assert.assertEquals(0, config.getAlertTimeout()); Assert.assertEquals(new IndexSpec(), config.getIndexSpec()); Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod()); Assert.assertEquals(NoneShardSpec.instance(), config.getShardSpec()); Assert.assertEquals(0, config.getMaxPendingPersists()); Assert.assertEquals(1000000, config.getMaxRowsInMemory()); Assert.assertEquals(0, config.getMergeThreadPriority()); Assert.assertEquals(0, config.getPersistThreadPriority()); Assert.assertEquals(new Period("PT10M"), config.getWindowPeriod()); Assert.assertEquals(false, config.isReportParseExceptions()); }
@Override public Appenderator build( final DataSchema schema, final RealtimeTuningConfig config, final FireDepartmentMetrics metrics ) { return Appenderators.createRealtime( schema, config.withBasePersistDirectory( makeBasePersistSubdirectory( config.getBasePersistDirectory(), schema.getDataSource(), config.getShardSpec() ) ), metrics, dataSegmentPusher, objectMapper, indexIO, indexMerger, conglomerate, segmentAnnouncer, emitter, queryExecutorService, cache, cacheConfig, cachePopulatorStats ); }
@Test public void testDefaultBasePersistDirectory() { final RealtimeTuningConfig tuningConfig1 = RealtimeTuningConfig.makeDefaultTuningConfig(null); final RealtimeTuningConfig tuningConfig2 = RealtimeTuningConfig.makeDefaultTuningConfig(null); Assert.assertNotEquals(tuningConfig1.getBasePersistDirectory(), tuningConfig2.getBasePersistDirectory()); }
final RealtimeTuningConfig defaults = RealtimeTuningConfig.makeDefaultTuningConfig(basePersistDirectory); this.maxRowsInMemory = maxRowsInMemory == null ? defaults.getMaxRowsInMemory() : maxRowsInMemory; this.maxRowsPerSegment = maxRowsPerSegment == null ? DEFAULT_MAX_ROWS_PER_SEGMENT : maxRowsPerSegment; this.maxTotalRows = maxTotalRows; ? defaults.getIntermediatePersistPeriod() : intermediatePersistPeriod; this.basePersistDirectory = defaults.getBasePersistDirectory(); this.maxPendingPersists = maxPendingPersists == null ? 0 : maxPendingPersists; this.indexSpec = indexSpec == null ? defaults.getIndexSpec() : indexSpec; this.reportParseExceptions = reportParseExceptions == null ? defaults.isReportParseExceptions() : reportParseExceptions; this.handoffConditionTimeout = handoffConditionTimeout == null ? defaults.getHandoffConditionTimeout() : handoffConditionTimeout; this.resetOffsetAutomatically = resetOffsetAutomatically == null
private void startPersistThread() final Period windowPeriod = config.getWindowPeriod(); "%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum() ); ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable =
schema.getAggregators(), fileToUpload, config.getIndexSpec(), config.getSegmentWriteOutMediumFactory() );
), new RealtimeTuningConfig( 1, null, new Period("PT10M"), task.getRealtimeIngestionSchema() .getTuningConfig().getWindowPeriod() ); Assert.assertEquals( task.getRealtimeIngestionSchema().getDataSchema().getGranularitySpec().getSegmentGranularity() ); Assert.assertTrue(task.getRealtimeIngestionSchema().getTuningConfig().isReportParseExceptions()); Assert.assertEquals(task.getTaskResource().getAvailabilityGroup(), task2.getTaskResource().getAvailabilityGroup()); Assert.assertEquals( task.getRealtimeIngestionSchema().getTuningConfig().getWindowPeriod(), task2.getRealtimeIngestionSchema().getTuningConfig().getWindowPeriod() ); Assert.assertEquals(
RealtimeIOConfig realtimeIOConfig = spec.getIOConfig(); RealtimeTuningConfig tuningConfig = spec.getTuningConfig() .withBasePersistDirectory(toolbox.getPersistDir()) .withVersioningPolicy(versioningPolicy); firehose, plumber, tuningConfig.isReportParseExceptions(), metrics );
private boolean runFirehose(Firehose firehose) { final Supplier<Committer> committerSupplier = Committers.supplierFromFirehose(firehose); while (firehose.hasMore()) { if (Thread.interrupted() || stopping) { return false; } Plumbers.addNextRow(committerSupplier, firehose, plumber, config.isReportParseExceptions(), metrics); } return true; }
@LifecycleStart public void start() { serverAnnouncer.announce(); fireChiefExecutor = Execs.multiThreaded(fireDepartments.size(), "chief-%d"); for (final FireDepartment fireDepartment : fireDepartments) { final DataSchema schema = fireDepartment.getDataSchema(); final FireChief chief = new FireChief(fireDepartment, conglomerate); chiefs.computeIfAbsent(schema.getDataSource(), k -> new HashMap<>()) .put(fireDepartment.getTuningConfig().getShardSpec().getPartitionNum(), chief); fireChiefExecutor.submit(chief); } }
protected File computeBaseDir(DataSchema schema) { return new File(config.getBasePersistDirectory(), schema.getDataSource()); }
private SegmentIdWithShardSpec getSegmentIdentifier(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final VersioningPolicy versioningPolicy = config.getVersioningPolicy(); DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp)); final long truncatedTime = truncatedDateTime.getMillis(); SegmentIdWithShardSpec retVal = segments.get(truncatedTime); if (retVal == null) { final Interval interval = new Interval( truncatedDateTime, segmentGranularity.increment(truncatedDateTime) ); retVal = new SegmentIdWithShardSpec( schema.getDataSource(), interval, versioningPolicy.getVersion(interval), config.getShardSpec() ); addSegment(retVal); } return retVal; }
private void resetNextFlush() { nextFlush = DateTimes.nowUtc().plus(config.getIntermediatePersistPeriod()).getMillis(); }
final long forceEndWaitTime = System.currentTimeMillis() + config.getHandoffConditionTimeout(); while (!sinks.isEmpty()) { try { if (config.getHandoffConditionTimeout() == 0) { handoffCondition.wait(); } else {
protected void startPersistThread() final Period windowPeriod = config.getWindowPeriod(); "%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum() ); ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable =
private void spillIfSwappable() { if (theSink.swappable()) { final FireHydrant indexToPersist = theSink.swap(); final int rowsToPersist = indexToPersist.getIndex().size(); final File dirToPersist = getSpillDir(indexToPersist.getCount()); log.info("Spilling index[%d] with rows[%d] to: %s", indexToPersist.getCount(), rowsToPersist, dirToPersist); try { indexMergerV9.persist( indexToPersist.getIndex(), dirToPersist, config.getIndexSpec(), config.getSegmentWriteOutMediumFactory() ); indexToPersist.swapSegment(null); metrics.incrementRowOutputCount(rowsToPersist); spilled.add(dirToPersist); } catch (Exception e) { log.warn(e, "Failed to spill index[%d]", indexToPersist.getCount()); throw Throwables.propagate(e); } } }
@Override public Appenderator build( final DataSchema schema, final RealtimeTuningConfig config, final FireDepartmentMetrics metrics ) { return Appenderators.createRealtime( schema, config.withBasePersistDirectory( makeBasePersistSubdirectory( config.getBasePersistDirectory(), schema.getDataSource(), config.getShardSpec() ) ), metrics, dataSegmentPusher, objectMapper, indexIO, indexMerger, conglomerate, segmentAnnouncer, emitter, queryExecutorService, cache, cacheConfig, cachePopulatorStats ); }
@Test public void testSpecificBasePersistDirectory() { final RealtimeTuningConfig tuningConfig = RealtimeTuningConfig.makeDefaultTuningConfig( new File("/tmp/nonexistent") ); Assert.assertEquals(new File("/tmp/nonexistent"), tuningConfig.getBasePersistDirectory()); }