@JsonCreator public KafkaTuningConfig(@JsonProperty("maxRowsInMemory") Integer maxRowsInMemory, @JsonProperty("maxRowsPerSegment") Integer maxRowsPerSegment, @JsonProperty("intermediatePersistPeriod") Period intermediatePersistPeriod, @JsonProperty("basePersistDirectory") File basePersistDirectory, @JsonProperty("maxPendingPersists") Integer maxPendingPersists, @JsonProperty("indexSpec") IndexSpec indexSpec, // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12. @SuppressWarnings("unused") @JsonProperty("buildV9Directly") Boolean buildV9Directly, @JsonProperty("reportParseExceptions") Boolean reportParseExceptions, @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout, @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically) { // Cannot be a static because default basePersistDirectory is unique per-instance final RealtimeTuningConfig defaults = RealtimeTuningConfig.makeDefaultTuningConfig(basePersistDirectory); this.maxRowsInMemory = maxRowsInMemory == null ? defaults.getMaxRowsInMemory() : maxRowsInMemory; this.maxRowsPerSegment = maxRowsPerSegment == null ? DEFAULT_MAX_ROWS_PER_SEGMENT : maxRowsPerSegment; this.intermediatePersistPeriod = intermediatePersistPeriod == null ? defaults.getIntermediatePersistPeriod() : intermediatePersistPeriod; this.basePersistDirectory = defaults.getBasePersistDirectory(); this.maxPendingPersists = maxPendingPersists == null ? defaults.getMaxPendingPersists() : maxPendingPersists; this.indexSpec = indexSpec == null ? defaults.getIndexSpec() : indexSpec; this.reportParseExceptions = reportParseExceptions == null ? defaults.isReportParseExceptions() : reportParseExceptions; this.handoffConditionTimeout = handoffConditionTimeout == null ? defaults.getHandoffConditionTimeout() : handoffConditionTimeout; this.resetOffsetAutomatically = resetOffsetAutomatically == null ? DEFAULT_RESET_OFFSET_AUTOMATICALLY : resetOffsetAutomatically; }
RealtimeTuningConfig realtimeTuningConfig = new RealtimeTuningConfig(maxRowInMemory, null, null,
public DruidRecordWriter(DataSchema dataSchema, RealtimeTuningConfig realtimeTuningConfig, DataSegmentPusher dataSegmentPusher, int maxPartitionSize, final Path segmentsDescriptorsDir, final FileSystem fileSystem) { File basePersistDir = new File(realtimeTuningConfig.getBasePersistDirectory(), UUID.randomUUID().toString()); this.tuningConfig = Preconditions.checkNotNull(realtimeTuningConfig.withBasePersistDirectory(basePersistDir), "realtimeTuningConfig is null"); this.dataSchema = Preconditions.checkNotNull(dataSchema, "data schema is null"); appenderator = Appenderators.createOffline(this.dataSchema, tuningConfig, new FireDepartmentMetrics(), dataSegmentPusher, DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.INDEX_IO, DruidStorageHandlerUtils.INDEX_MERGER_V9); this.maxPartitionSize = maxPartitionSize; appenderator.startJob(); this.segmentsDescriptorDir = Preconditions.checkNotNull(segmentsDescriptorsDir, "segmentsDescriptorsDir is null"); this.fileSystem = Preconditions.checkNotNull(fileSystem, "file system is null"); this.segmentGranularity = this.dataSchema.getGranularitySpec().getSegmentGranularity(); committerSupplier = Suppliers.ofInstance(Committers.nil())::get; }
private Sink getSink(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final VersioningPolicy versioningPolicy = config.getVersioningPolicy(); DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp)); final long truncatedTime = truncatedDateTime.getMillis(); Sink retVal = sinks.get(truncatedTime); if (retVal == null) { final Interval sinkInterval = new Interval( truncatedDateTime, segmentGranularity.increment(truncatedDateTime) ); retVal = new Sink( sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getMaxRowsInMemory(), config.isReportParseExceptions() ); addSink(retVal); } return retVal; }
schema.getAggregators(), fileToUpload, config.getIndexSpec(), config.getSegmentWriteOutMediumFactory() );
interval, schema, config.getShardSpec(), version, config.getMaxRowsInMemory(), config.isReportParseExceptions() );
@Override public Appenderator build( final DataSchema schema, final RealtimeTuningConfig config, final FireDepartmentMetrics metrics ) { return Appenderators.createRealtime( schema, config.withBasePersistDirectory( makeBasePersistSubdirectory( config.getBasePersistDirectory(), schema.getDataSource(), config.getShardSpec() ) ), metrics, dataSegmentPusher, objectMapper, indexIO, indexMerger, conglomerate, segmentAnnouncer, emitter, queryExecutorService, cache, cacheConfig ); }
RealtimeIOConfig realtimeIOConfig = spec.getIOConfig(); RealtimeTuningConfig tuningConfig = spec.getTuningConfig() .withBasePersistDirectory(toolbox.getPersistDir()) .withVersioningPolicy(versioningPolicy); firehose, plumber, tuningConfig.isReportParseExceptions(), metrics );
@Override public void close(boolean abort) throws IOException { try { if (!abort) { final List<SegmentIdentifier> segmentsToPush = Lists.newArrayList(); segmentsToPush.addAll(appenderator.getSegments()); pushSegments(segmentsToPush); } appenderator.clear(); } catch (InterruptedException e) { Throwables.propagate(e); } finally { try { FileUtils.deleteDirectory(tuningConfig.getBasePersistDirectory()); } catch (Exception e) { LOG.error("error cleaning of base persist directory", e); } appenderator.close(); } }
protected void initializeExecutors() { final int maxPendingPersists = config.getMaxPendingPersists(); if (persistExecutor == null) { // use a blocking single threaded executor to throttle the firehose when write to disk is slow persistExecutor = Execs.newBlockingSingleThreaded( "plumber_persist_%d", maxPendingPersists, TaskThreadPriority.getThreadPriorityFromTaskPriority(config.getPersistThreadPriority()) ); } if (mergeExecutor == null) { // use a blocking single threaded executor to throttle the firehose when write to disk is slow mergeExecutor = Execs.newBlockingSingleThreaded( "plumber_merge_%d", 1, TaskThreadPriority.getThreadPriorityFromTaskPriority(config.getMergeThreadPriority()) ); } if (scheduledExecutor == null) { scheduledExecutor = Execs.scheduledSingleThreaded("plumber_scheduled_%d"); } }
private boolean runFirehose(Firehose firehose) { final Supplier<Committer> committerSupplier = Committers.supplierFromFirehose(firehose); while (firehose.hasMore()) { if (Thread.interrupted() || stopping) { return false; } Plumbers.addNextRow(committerSupplier, firehose, plumber, config.isReportParseExceptions(), metrics); } return true; }
@JsonCreator public FireDepartment( @JsonProperty("dataSchema") DataSchema dataSchema, @JsonProperty("ioConfig") RealtimeIOConfig ioConfig, @JsonProperty("tuningConfig") RealtimeTuningConfig tuningConfig ) { super(dataSchema, ioConfig, tuningConfig); Preconditions.checkNotNull(dataSchema, "dataSchema"); Preconditions.checkNotNull(ioConfig, "ioConfig"); this.dataSchema = dataSchema; this.ioConfig = ioConfig; this.tuningConfig = tuningConfig == null ? RealtimeTuningConfig.makeDefaultTuningConfig(null) : tuningConfig; }
private void resetNextFlush() { nextFlush = DateTimes.nowUtc().plus(config.getIntermediatePersistPeriod()).getMillis(); }
final long forceEndWaitTime = System.currentTimeMillis() + config.getHandoffConditionTimeout(); while (!sinks.isEmpty()) { try { if (config.getHandoffConditionTimeout() == 0) { handoffCondition.wait(); } else {
protected Object bootstrapSinksFromDisk() final VersioningPolicy versioningPolicy = config.getVersioningPolicy(); sinkInterval.getEnd(), versioningPolicy.getVersion(sinkInterval), config.getShardSpec() ), queryableIndex sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getMaxRowsInMemory(), config.isReportParseExceptions(), hydrants );
private void spillIfSwappable() { if (theSink.swappable()) { final FireHydrant indexToPersist = theSink.swap(); final int rowsToPersist = indexToPersist.getIndex().size(); final File dirToPersist = getSpillDir(indexToPersist.getCount()); log.info("Spilling index[%d] with rows[%d] to: %s", indexToPersist.getCount(), rowsToPersist, dirToPersist); try { indexMergerV9.persist( indexToPersist.getIndex(), dirToPersist, config.getIndexSpec(), config.getSegmentWriteOutMediumFactory() ); indexToPersist.swapSegment(null); metrics.incrementRowOutputCount(rowsToPersist); spilled.add(dirToPersist); } catch (Exception e) { log.warn(e, "Failed to spill index[%d]", indexToPersist.getCount()); throw Throwables.propagate(e); } } }
protected File computeBaseDir(DataSchema schema) { return new File(config.getBasePersistDirectory(), schema.getDataSource()); }
final RealtimeTuningConfig defaults = RealtimeTuningConfig.makeDefaultTuningConfig(basePersistDirectory); this.maxRowsInMemory = maxRowsInMemory == null ? defaults.getMaxRowsInMemory() : maxRowsInMemory; this.maxRowsPerSegment = maxRowsPerSegment == null ? DEFAULT_MAX_ROWS_PER_SEGMENT : maxRowsPerSegment; this.intermediatePersistPeriod = intermediatePersistPeriod == null ? defaults.getIntermediatePersistPeriod() : intermediatePersistPeriod; this.basePersistDirectory = defaults.getBasePersistDirectory(); this.maxPendingPersists = 0; this.indexSpec = indexSpec == null ? defaults.getIndexSpec() : indexSpec; this.reportParseExceptions = reportParseExceptions == null ? defaults.isReportParseExceptions() : reportParseExceptions; this.handoffConditionTimeout = handoffConditionTimeout == null ? defaults.getHandoffConditionTimeout() : handoffConditionTimeout; this.resetOffsetAutomatically = resetOffsetAutomatically == null
int numRows = indexToPersist.getIndex().size(); final IndexSpec indexSpec = config.getIndexSpec(); new File(computePersistDir(schema, interval), String.valueOf(indexToPersist.getCount())), indexSpec, config.getSegmentWriteOutMediumFactory() );
RealtimeTuningConfig tuningConfig = new RealtimeTuningConfig(null, null, null,