private void removeSegment(DataSegment segment) { totalSegmentSize -= segment.getSize(); numSegments--; }
private void addSegment(DataSegment segment) { totalSegmentSize += segment.getSize(); numSegments++; }
public long getSegmentSize() { return segment.getSize(); }
public long getSegmentSize() { return segment.getSize(); }
synchronized void addSegment(DataSegment segment) { if (segments.add(segment)) { currSize += segment.getSize(); } }
synchronized void addSegment(DataSegment segment) { if (segments.add(segment)) { currSize += segment.getSize(); } }
synchronized void removeSegment(DataSegment segment) { if (segments.remove(segment)) { currSize -= segment.getSize(); } }
synchronized void removeSegment(DataSegment segment) { if (segments.remove(segment)) { currSize -= segment.getSize(); } }
@Override public long getLength() throws IOException, InterruptedException { long size = 0; for (WindowedDataSegment segment : segments) { size += segment.getSegment().getSize(); } return size; }
public LoadSegmentHolder(DataSegment segment, LoadPeonCallback callback) { super(segment, new SegmentChangeRequestLoad(segment), callback); queuedSize.addAndGet(segment.getSize()); }
@Override public void requestSucceeded() { queuedSize.addAndGet(-getSegment().getSize()); super.requestSucceeded(); } }
private void setProperties( final Map<Interval, Map<String, Map<String, Object>>> retVal, ImmutableDruidDataSource dataSource, DataSegment dataSegment ) { Map<String, Object> properties = retVal.get(dataSegment.getInterval()).get(dataSource.getName()); if (properties == null) { properties = Maps.newHashMap(); properties.put("size", dataSegment.getSize()); properties.put("count", 1); retVal.get(dataSegment.getInterval()).put(dataSource.getName(), properties); } else { properties.put("size", MapUtils.getLong(properties, "size", 0L) + dataSegment.getSize()); properties.put("count", MapUtils.getInt(properties, "count", 0) + 1); } } }
public Set<DataSegment> getOrderedAvailableDataSegments() { Set<DataSegment> availableSegments = Sets.newTreeSet(SEGMENT_COMPARATOR); Iterable<DataSegment> dataSegments = getAvailableDataSegments(); for (DataSegment dataSegment : dataSegments) { if (dataSegment.getSize() < 0) { log.makeAlert("No size on Segment, wtf?") .addData("segment", dataSegment) .emit(); } availableSegments.add(dataSegment); } return availableSegments; }
private void loadInLocation(DataSegment segment, File storageDir) throws SegmentLoadingException { // LoadSpec isn't materialized until here so that any system can interpret Segment without having to have all the // LoadSpec dependencies. final LoadSpec loadSpec = jsonMapper.convertValue(segment.getLoadSpec(), LoadSpec.class); final LoadSpec.LoadSpecResult result = loadSpec.loadSegment(storageDir); if (result.getSize() != segment.getSize()) { log.warn( "Segment [%s] is different than expected size. Expected [%d] found [%d]", segment.getIdentifier(), segment.getSize(), result.getSize() ); } }
@Override protected double computeCost(DataSegment proposalSegment, ServerHolder server, boolean includeCurrentServer) { final long proposalSegmentSize = proposalSegment.getSize(); // (optional) Don't include server if it is already serving segment if (!includeCurrentServer && server.isServingSegment(proposalSegment)) { return Double.POSITIVE_INFINITY; } // Don't calculate cost if the server doesn't have enough space or is loading the segment if (proposalSegmentSize > server.getAvailableSize() || server.isLoadingSegment(proposalSegment)) { return Double.POSITIVE_INFINITY; } final String serverName = server.getServer().getName(); double cost = clusterCostCache.computeCost(serverName, proposalSegment); // add segments that will be loaded to the cost cost += costCacheForLoadingSegments(server).computeCost(serverName, proposalSegment); return cost; }
/** * Given a druid data segment constructs an object to hold the information of this partition. * * @param segment The druid data segments that corresponds to a specific partition of a druid segment. */ public SegmentInfo(DataSegment segment) { this.dataSource = segment.getDataSource(); this.interval = segment.getInterval(); this.dimensions = segment.getDimensions(); this.metrics = segment.getMetrics(); this.version = segment.getVersion(); ShardSpec spec = segment.getShardSpec(); this.shardSpec = spec instanceof NumberedShardSpec ? (NumberedShardSpec) spec : new NumberedShardSpec((NoneShardSpec) spec); this.size = segment.getSize(); this.identifier = segment.getIdentifier(); }
@Override public Void perform(Task task, TaskActionToolbox toolbox) throws IOException { toolbox.verifyTaskLocksAndSinglePartitionSettitude(task, segments, true); toolbox.getIndexerDBCoordinator().deleteSegments(segments); // Emit metrics final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder() .setUser2(task.getDataSource()) .setUser4(task.getType()); for (DataSegment segment : segments) { metricBuilder.setUser5(segment.getInterval().toString()); toolbox.getEmitter().emit(metricBuilder.build("indexer/segmentNuked/bytes", segment.getSize())); } return null; }
@Override public Set<DataSegment> perform(Task task, TaskActionToolbox toolbox) throws IOException { toolbox.verifyTaskLocksAndSinglePartitionSettitude(task, segments, true); final Set<DataSegment> retVal = toolbox.getIndexerDBCoordinator().announceHistoricalSegments(segments); // Emit metrics final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder() .setUser2(task.getDataSource()) .setUser4(task.getType()); for (DataSegment segment : segments) { metricBuilder.setUser5(segment.getInterval().toString()); toolbox.getEmitter().emit(metricBuilder.build("indexer/segment/bytes", segment.getSize())); } return retVal; }
@Override public Void perform( Task task, TaskActionToolbox toolbox ) throws IOException { toolbox.verifyTaskLocksAndSinglePartitionSettitude(task, segments, true); toolbox.getIndexerDBCoordinator().updateSegmentMetadata(segments); // Emit metrics final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder() .setUser2(task.getDataSource()) .setUser4(task.getType()); for (DataSegment segment : segments) { metricBuilder.setUser5(segment.getInterval().toString()); toolbox.getEmitter().emit(metricBuilder.build("indexer/segmentMoved/bytes", segment.getSize())); } return null; }