final Granularity segmentGranularity = config.getGranularitySpec().getSegmentGranularity(); new UniformGranularitySpec( segmentGranularity, config.getGranularitySpec().getQueryGranularity(), config.getGranularitySpec().isRollup(), Lists.newArrayList(bucketsToRun)
int thrownAway = 0; int unparseable = 0; final Granularity queryGranularity = granularitySpec.getQueryGranularity(); interval = granularitySpec.getSegmentGranularity().bucket(inputRow.getTimestamp()); } else { if (!Intervals.ETERNITY.contains(inputRow.getTimestamp())) { final Optional<Interval> optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp()); if (!optInterval.isPresent()) { determinePartitionsMeters.incrementThrownAway();
final boolean explicitIntervals = dataSchema.getGranularitySpec().bucketIntervals().isPresent(); final ParallelIndexIOConfig ioConfig = ingestionSchema.getIOConfig(); if (ioConfig.isAppendToExisting() || !explicitIntervals) { schema.getDataSource(), row.getTimestamp(), schema.getGranularitySpec().getQueryGranularity(), schema.getGranularitySpec().getSegmentGranularity(), sequenceName, previousSegmentId,
final GranularitySpec granularitySpec = getIngestionSchema().getDataSchema().getGranularitySpec(); final SortedSet<Interval> bucketIntervals = Preconditions.checkNotNull( granularitySpec.bucketIntervals().orNull(), "bucketIntervals" ); .collect(Collectors.toMap(TaskLock::getInterval, TaskLock::getVersion)); final Optional<Interval> maybeInterval = granularitySpec.bucketInterval(timestamp); if (!maybeInterval.isPresent()) { throw new IAE("Could not find interval for timestamp [%s]", timestamp);
interval = config.getGranularitySpec().getSegmentGranularity().bucket(DateTimes.utc(key.get())); } else { Optional<Interval> intervalOptional = config.getGranularitySpec().bucketInterval(DateTimes.utc(key.get()));
private DateTime getAllowedMinTime() { final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final Period windowPeriod = config.getWindowPeriod(); final long windowMillis = windowPeriod.toStandardDuration().getMillis(); return segmentGranularity.bucketStart( DateTimes.utc(Math.max(windowMillis, rejectionPolicy.getCurrMaxTime().getMillis()) - windowMillis) ); }
.withMinTimestamp(minTimestamp) .withTimestampSpec(schema.getParser()) .withQueryGranularity(schema.getGranularitySpec().getQueryGranularity()) .withDimensionsSpec(schema.getParser()) .withMetrics(schema.getAggregators()) .withRollup(schema.getGranularitySpec().isRollup()) .build(); final IncrementalIndex newIndex = new IncrementalIndex.Builder()
public StreamAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, final FireDepartmentMetrics metrics ) { return new StreamAppenderatorDriver( appenderator, new ActionBasedSegmentAllocator( toolbox.getTaskActionClient(), dataSchema, (schema, row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> new SegmentAllocateAction( schema.getDataSource(), row.getTimestamp(), schema.getGranularitySpec().getQueryGranularity(), schema.getGranularitySpec().getSegmentGranularity(), sequenceName, previousSegmentId, skipSegmentLineageCheck ) ), toolbox.getSegmentHandoffNotifierFactory(), new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), toolbox.getDataSegmentKiller(), toolbox.getObjectMapper(), metrics ); }
Assert.assertTrue(spec.isRollup()); Intervals.of("2012-02-01T00Z/2012-03-01T00Z") ), Lists.newArrayList(spec.bucketIntervals().get()) ); "2012-01-03T00Z", Optional.of(Intervals.of("2012-01-03T00Z/2012-01-04T00Z")), spec.bucketInterval(DateTimes.of("2012-01-03T00Z")) ); "2012-01-03T01Z", Optional.of(Intervals.of("2012-01-03T00Z/2012-01-04T00Z")), spec.bucketInterval(DateTimes.of("2012-01-03T01Z")) ); "2012-01-04T01Z", Optional.<Interval>absent(), spec.bucketInterval(DateTimes.of("2012-01-04T01Z")) ); "2012-01-07T23:59:59.999Z", Optional.of(Intervals.of("2012-01-07T00Z/2012-01-08T00Z")), spec.bucketInterval(DateTimes.of("2012-01-07T23:59:59.999Z")) ); "2012-01-08T01Z", Optional.of(Intervals.of("2012-01-08T00Z/2012-01-11T00Z")),
} else if (this.granularitySpec.isRollup()) { log.warn("No metricsSpec has been specified. Are you sure this is what you want?");
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); rollupGranularity = getConfig().getGranularitySpec().getQueryGranularity(); config = HadoopDruidIndexerConfig.fromConfiguration(context.getConfiguration()); Optional<Set<Interval>> intervals = config.getSegmentGranularIntervals(); if (intervals.isPresent()) { determineIntervals = false; final ImmutableMap.Builder<Interval, HyperLogLogCollector> builder = ImmutableMap.builder(); for (final Interval bucketInterval : intervals.get()) { builder.put(bucketInterval, HyperLogLogCollector.makeLatestCollector()); } hyperLogLogs = builder.build(); } else { determineIntervals = true; hyperLogLogs = new HashMap<>(); } }
public DeterminePartitionsDimSelectionMapperHelper(HadoopDruidIndexerConfig config, String partitionDimension) { this.config = config; this.partitionDimension = partitionDimension; final ImmutableMap.Builder<Long, Integer> timeIndexBuilder = ImmutableMap.builder(); int idx = 0; for (final Interval bucketInterval : config.getGranularitySpec().bucketIntervals().get()) { timeIndexBuilder.put(bucketInterval.getStartMillis(), idx); idx++; } this.intervalIndexes = timeIndexBuilder.build(); }
@Test public void testJson() { final GranularitySpec spec = new UniformGranularitySpec( Granularities.DAY, null, Lists.newArrayList( Intervals.of("2012-01-08T00Z/2012-01-11T00Z"), Intervals.of("2012-01-07T00Z/2012-01-08T00Z"), Intervals.of("2012-01-03T00Z/2012-01-04T00Z"), Intervals.of("2012-01-01T00Z/2012-01-03T00Z") ) ); try { final GranularitySpec rtSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(spec), GranularitySpec.class); Assert.assertEquals( "Round-trip bucketIntervals", spec.bucketIntervals(), rtSpec.bucketIntervals() ); Assert.assertEquals( "Round-trip granularity", spec.getSegmentGranularity(), rtSpec.getSegmentGranularity() ); } catch (Exception e) { throw Throwables.propagate(e); } }
) throws IOException, InterruptedException final Optional<Interval> maybeInterval = config.getGranularitySpec().bucketInterval(timestamp);
public List<Interval> getInputIntervals() { return schema.getDataSchema() .getGranularitySpec() .inputIntervals(); }
.bucketIntervals() .isPresent(); ingestionSchema.getDataSchema() .getGranularitySpec() .withIntervals( JodaUtils.condenseIntervals( shardSpecs.getIntervals()
private void startPersistThread() final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final Period windowPeriod = config.getWindowPeriod();
if (determineIntervals) { interval = config.getGranularitySpec() .getSegmentGranularity() .bucket(DateTimes.utc(inputRow.getTimestampFromEpoch())); .bucketInterval(DateTimes.utc(inputRow.getTimestampFromEpoch()));
private static IncrementalIndex makeIncrementalIndex( Bucket theBucket, AggregatorFactory[] aggs, HadoopDruidIndexerConfig config, Iterable<String> oldDimOrder, Map<String, ColumnCapabilitiesImpl> oldCapabilities ) { final HadoopTuningConfig tuningConfig = config.getSchema().getTuningConfig(); final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder() .withMinTimestamp(theBucket.time.getMillis()) .withTimestampSpec(config.getSchema().getDataSchema().getParser().getParseSpec().getTimestampSpec()) .withDimensionsSpec(config.getSchema().getDataSchema().getParser()) .withQueryGranularity(config.getSchema().getDataSchema().getGranularitySpec().getQueryGranularity()) .withMetrics(aggs) .withRollup(config.getSchema().getDataSchema().getGranularitySpec().isRollup()) .build(); IncrementalIndex newIndex = new IncrementalIndex.Builder() .setIndexSchema(indexSchema) .setReportParseExceptions(!tuningConfig.isIgnoreInvalidRows()) // only used by OffHeapIncrementalIndex .setMaxRowCount(tuningConfig.getRowFlushBoundary()) .setMaxBytesInMemory(TuningConfigs.getMaxBytesInMemoryOrDefault(tuningConfig.getMaxBytesInMemory())) .buildOnheap(); if (oldDimOrder != null && !indexSchema.getDimensionsSpec().hasCustomDimensions()) { newIndex.loadDimensionIterable(oldDimOrder, oldCapabilities); } return newIndex; }
private static StreamAppenderatorDriver newDriver( final DataSchema dataSchema, final Appenderator appenderator, final TaskToolbox toolbox, final FireDepartmentMetrics metrics ) { return new StreamAppenderatorDriver( appenderator, new ActionBasedSegmentAllocator( toolbox.getTaskActionClient(), dataSchema, (schema, row, sequenceName, previousSegmentId, skipSegmentLineageCheck) -> new SegmentAllocateAction( schema.getDataSource(), row.getTimestamp(), schema.getGranularitySpec().getQueryGranularity(), schema.getGranularitySpec().getSegmentGranularity(), sequenceName, previousSegmentId, skipSegmentLineageCheck ) ), toolbox.getSegmentHandoffNotifierFactory(), new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), toolbox.getDataSegmentKiller(), toolbox.getObjectMapper(), metrics ); }