/** * Utility method which computes total memory by adding * FieldData, PercolatorCache, Segments (memory, index writer, version map) */ public ByteSizeValue getTotalMemory() { long size = 0; if (this.getFieldData() != null) { size += this.getFieldData().getMemorySizeInBytes(); } if (this.getQueryCache() != null) { size += this.getQueryCache().getMemorySizeInBytes(); } if (this.getSegments() != null) { size += this.getSegments().getMemoryInBytes() + this.getSegments().getIndexWriterMemoryInBytes() + this.getSegments().getVersionMapMemoryInBytes(); } return new ByteSizeValue(size); }
public void add(SegmentsStats mergeStats) { if (mergeStats == null) { return; } updateMaxUnsafeAutoIdTimestamp(mergeStats.maxUnsafeAutoIdTimestamp); add(mergeStats.count, mergeStats.memoryInBytes); addTermsMemoryInBytes(mergeStats.termsMemoryInBytes); addStoredFieldsMemoryInBytes(mergeStats.storedFieldsMemoryInBytes); addTermVectorsMemoryInBytes(mergeStats.termVectorsMemoryInBytes); addNormsMemoryInBytes(mergeStats.normsMemoryInBytes); addPointsMemoryInBytes(mergeStats.pointsMemoryInBytes); addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes); addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes); addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes); addBitsetMemoryInBytes(mergeStats.bitsetMemoryInBytes); addFileSizes(mergeStats.fileSizes); }
protected void writerSegmentStats(SegmentsStats stats) { // by default we don't have a writer here... subclasses can override this stats.addVersionMapMemoryInBytes(0); stats.addIndexWriterMemoryInBytes(0); }
@Override protected final void writerSegmentStats(SegmentsStats stats) { stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); stats.updateMaxUnsafeAutoIdTimestamp(maxUnsafeAutoIdTimestamp.get()); }
private void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) { stats.add(1, segmentReader.ramBytesUsed()); stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); if (includeSegmentFileSizes) { // TODO: consider moving this to StoreStats stats.addFileSizes(getSegmentFileSizes(segmentReader)); } }
public void add(SegmentsStats mergeStats) { if (mergeStats == null) { return; } add(mergeStats.count, mergeStats.memoryInBytes); addTermsMemoryInBytes(mergeStats.termsMemoryInBytes); addStoredFieldsMemoryInBytes(mergeStats.storedFieldsMemoryInBytes); addTermVectorsMemoryInBytes(mergeStats.termVectorsMemoryInBytes); addNormsMemoryInBytes(mergeStats.normsMemoryInBytes); addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes); addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes); addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes); addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes); addBitsetMemoryInBytes(mergeStats.bitsetMemoryInBytes); }
/** * Global stats on segments. */ public final SegmentsStats segmentsStats() { ensureOpen(); try (final Searcher searcher = acquireSearcher("segments_stats", false)) { SegmentsStats stats = new SegmentsStats(); for (LeafReaderContext reader : searcher.reader().leaves()) { final SegmentReader segmentReader = segmentReader(reader.reader()); stats.add(1, segmentReader.ramBytesUsed()); stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); } writerSegmentStats(stats); return stats; } }
this.queryCache = new QueryCacheStats(); this.completion = new CompletionStats(); this.segments = new SegmentsStats(); queryCache.add(shardCommonStats.queryCache); completion.add(shardCommonStats.completion); segments.add(shardCommonStats.segments);
protected void writerSegmentStats(SegmentsStats stats) { // by default we don't have a writer here... subclasses can override this stats.addVersionMapMemoryInBytes(0); stats.addIndexWriterMemoryInBytes(0); stats.addIndexWriterMaxMemoryInBytes(0); }
/** * Global stats on segments. */ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes) { ensureOpen(); Set<String> segmentName = new HashSet<>(); SegmentsStats stats = new SegmentsStats(); try (Searcher searcher = acquireSearcher("segments_stats", SearcherScope.INTERNAL)) { for (LeafReaderContext ctx : searcher.reader().getContext().leaves()) { SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); fillSegmentStats(segmentReader, includeSegmentFileSizes, stats); segmentName.add(segmentReader.getSegmentName()); } } try (Searcher searcher = acquireSearcher("segments_stats", SearcherScope.EXTERNAL)) { for (LeafReaderContext ctx : searcher.reader().getContext().leaves()) { SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); if (segmentName.contains(segmentReader.getSegmentName()) == false) { fillSegmentStats(segmentReader, includeSegmentFileSizes, stats); } } } writerSegmentStats(stats); return stats; }
@Override protected final void writerSegmentStats(SegmentsStats stats) { //stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); stats.updateMaxUnsafeAutoIdTimestamp(maxUnsafeAutoIdTimestamp.get()); }
public SegmentsStats segmentStats(boolean includeSegmentFileSizes) { SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes); segmentsStats.addBitsetMemoryInBytes(shardBitsetFilterCache.getMemorySizeInBytes()); return segmentsStats; }
public static SegmentsStats readSegmentsStats(StreamInput in) throws IOException { SegmentsStats stats = new SegmentsStats(); stats.readFrom(in); return stats; }
@Override public void ensureEstimatedStats() { if (size() > 0) { NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats() .clear().setBreaker(true).setIndices(true).execute().actionGet(); for (NodeStats stats : nodeStats.getNodes()) { assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(), stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L)); assertThat("Accounting breaker not reset to " + stats.getIndices().getSegments().getMemoryInBytes() + " on node: " + stats.getNode(), stats.getBreaker().getStats(CircuitBreaker.ACCOUNTING).getEstimated(), equalTo(stats.getIndices().getSegments().getMemoryInBytes())); // ExternalTestCluster does not check the request breaker, // because checking it requires a network request, which in // turn increments the breaker, making it non-0 assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); assertThat("Query cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L)); assertThat("FixedBitSet cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getSegments().getBitsetMemoryInBytes(), equalTo(0L)); } } }
public SegmentsStats segmentStats(boolean includeSegmentFileSizes) { SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes); segmentsStats.addBitsetMemoryInBytes(shardBitsetFilterCache.getMemorySizeInBytes()); segmentsStats.addTokenRangesBitsetMemoryInBytes(tokenRangesBitsetFilterCache.getMemorySizeInBytes()); return segmentsStats; }
private void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) { stats.add(1, segmentReader.ramBytesUsed()); stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); if (includeSegmentFileSizes) { // TODO: consider moving this to StoreStats stats.addFileSizes(getSegmentFileSizes(segmentReader)); } }
segments = new SegmentsStats(); segments.add(stats.getSegments()); segments.add(stats.getSegments());
@Override protected final void writerSegmentStats(SegmentsStats stats) { stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); stats.updateMaxUnsafeAutoIdTimestamp(maxUnsafeAutoIdTimestamp.get()); }
protected void writerSegmentStats(SegmentsStats stats) { // by default we don't have a writer here... subclasses can override this stats.addVersionMapMemoryInBytes(0); stats.addIndexWriterMemoryInBytes(0); }
@Override protected final void writerSegmentStats(SegmentsStats stats) { stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); stats.addIndexWriterMaxMemoryInBytes((long) (indexWriter.getConfig().getRAMBufferSizeMB() * 1024 * 1024)); }