@Override public MetricData GetDataByPoints(String tenantId, String metric, long from, long to, int points) throws SerializationException { rollupsByPointsMeter.mark(); Granularity g = Granularity.granularityFromPointsInInterval(tenantId, from, to, points); return getRollupByGranularity(tenantId, Arrays.asList(metric), from, to, g).get(Locator.createLocatorFromPathComponents(tenantId, metric)); }
private String bulkStringify(List<IMetric> metrics){ StringBuilder sb = new StringBuilder(); for(IMetric metric : metrics){ Locator locator = metric.getLocator(); if(locator.getMetricName() == null) throw new IllegalArgumentException("trying to insert metric discovery without a metricName"); sb.append(String.format( "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\", \"routing\" : \"%s\" } }%n", AbstractElasticIO.ELASTICSEARCH_INDEX_NAME_WRITE, AbstractElasticIO.ELASTICSEARCH_DOCUMENT_TYPE, locator.getTenantId() + ":" + locator.getMetricName(), locator.getTenantId())); if(metric instanceof Metric && getUnit((Metric) metric) != null){ sb.append(String.format( "{ \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\" }%n", ESFieldLabel.tenantId.toString(), locator.getTenantId(), ESFieldLabel.metric_name.toString(), locator.getMetricName(), ESFieldLabel.unit.toString(), getUnit((Metric) metric))); } else { sb.append(String.format( "{ \"%s\" : \"%s\", \"%s\" : \"%s\" }%n", ESFieldLabel.tenantId.toString(), locator.getTenantId(), ESFieldLabel.metric_name.toString(), locator.getMetricName())); } } return sb.toString(); }
/** * This method is used to log delayed metrics, if tracking delayed metrics * is turned on for this Blueflood service. * @param tenantid * @param delayedMetrics */ public void trackDelayedMetricsTenant(String tenantid, final List<Metric> delayedMetrics) { if (isTrackingDelayedMetrics) { String logMessage = String.format("[TRACKER][DELAYED METRIC] Tenant sending delayed metrics %s", tenantid); log.info(logMessage); // log individual delayed metrics locator and collectionTime double delayedMinutes; long nowMillis = System.currentTimeMillis(); for (Metric metric : delayedMetrics) { delayedMinutes = (double)(nowMillis - metric.getCollectionTime()) / 1000 / 60; logMessage = String.format("[TRACKER][DELAYED METRIC] %s has collectionTime %s which is delayed by %.2f minutes", metric.getLocator().toString(), dateFormatter.format(new Date(metric.getCollectionTime())), delayedMinutes); log.info(logMessage); } } }
@Override public JSONObject transformRollupData(Map<Locator, MetricData> metricData, Set<MetricStat> filterStats) throws SerializationException { final JSONObject globalJSON = new JSONObject(); final JSONArray metricsArray = new JSONArray(); for (Map.Entry<Locator, MetricData> one : metricData.entrySet()) { final JSONObject singleMetricJSON = new JSONObject(); singleMetricJSON.put("metric", one.getKey().getMetricName()); singleMetricJSON.put("unit", one.getValue().getUnit() == null ? Util.UNKNOWN : one.getValue().getUnit()); singleMetricJSON.put("type", one.getValue().getType()); Set<MetricStat> oneFilterStats = fixFilterStats(one.getValue(), filterStats); JSONArray values = transformDataToJSONArray(one.getValue(), oneFilterStats); singleMetricJSON.put("data", values); metricsArray.add(singleMetricJSON); } globalJSON.put("metrics", metricsArray); return globalJSON; } }
checkFromJson.getMonitoringZoneId(), metricName).trim(); metricLocator = Locator.createLocatorFromDbKey(longMetricName); } else { String longMetricName = String.format("%s.rackspace.monitoring.entities.%s.checks.%s.%s.%s", checkFromJson.getCheckId(), metricName).trim(); metricLocator = Locator.createLocatorFromDbKey(longMetricName); if (!shardsToBackfill.contains(Util.computeShard(metricLocator.toString()))) continue;
private String bulkStringifyTokens(List<Token> tokens){ StringBuilder sb = new StringBuilder(); for(Token token : tokens){ sb.append(String.format( "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\", \"routing\" : \"%s\" } }%n", ElasticTokensIO.ELASTICSEARCH_TOKEN_INDEX_NAME_WRITE, ElasticTokensIO.ES_DOCUMENT_TYPE, token.getId(), token.getLocator().getTenantId())); sb.append(String.format( "{ \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\" }%n", ESFieldLabel.token.toString(), token.getToken(), ESFieldLabel.parent.toString(), token.getParent(), ESFieldLabel.isLeaf.toString(), token.isLeaf(), ESFieldLabel.tenantId.toString(), token.getLocator().getTenantId())); } return sb.toString(); }
protected SearchResult createExpectedResult(String tenantId, int x, String y, int z, String unit) { Locator locator = createTestLocator(tenantId, x, y, z); return new SearchResult(tenantId, locator.getMetricName(), unit); }
private String bulkStringifyTokens(List<Token> tokens, String indexName){ StringBuilder sb = new StringBuilder(); for(Token token : tokens){ sb.append(String.format( "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\", \"routing\" : \"%s\" } }%n", indexName, ElasticTokensIO.ES_DOCUMENT_TYPE, token.getId(), token.getLocator().getTenantId())); sb.append(String.format( "{ \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\", \"%s\" : \"%s\" }%n", ESFieldLabel.token.toString(), token.getToken(), ESFieldLabel.parent.toString(), token.getParent(), ESFieldLabel.isLeaf.toString(), token.isLeaf(), ESFieldLabel.tenantId.toString(), token.getLocator().getTenantId())); } return sb.toString(); }
@Override public MetricData GetDataByResolution(String tenantId, String metric, long from, long to, Resolution resolution) throws SerializationException { rollupsByGranularityMeter.mark(); if (resolution == null) { resolution = Resolution.FULL; } Granularity g = Granularity.granularities()[resolution.getValue()]; return getRollupByGranularity(tenantId, Arrays.asList(metric), from, to, g).get(Locator.createLocatorFromPathComponents(tenantId, metric)); }
public void testWildcard(String tenantId, String unit) throws Exception { SearchResult entry; List<SearchResult> results; results = elasticIO.search(tenantId, "one.two.*"); List<Locator> locators = locatorMap.get(tenantId); assertEquals(locators.size(), results.size()); for (Locator locator : locators) { entry = new SearchResult(tenantId, locator.getMetricName(), unit); Assert.assertTrue((results.contains(entry))); } results = elasticIO.search(tenantId, "*.fourA.*"); assertEquals(NUM_PARENT_ELEMENTS * NUM_GRANDCHILD_ELEMENTS, results.size()); for (int x = 0; x < NUM_PARENT_ELEMENTS; x++) { for (int z = 0; z < NUM_GRANDCHILD_ELEMENTS; z++) { entry = createExpectedResult(tenantId, x, "A", z, unit); Assert.assertTrue(results.contains(entry)); } } results = elasticIO.search(tenantId, "*.three1*.four*.five2"); assertEquals(10 * CHILD_ELEMENTS.size(), results.size()); for (int x = 10; x < 20; x++) { for (String y : CHILD_ELEMENTS) { entry = createExpectedResult(tenantId, x, y, 2, unit); Assert.assertTrue(results.contains(entry)); } } }
@Test public void testTrackDelayedAggregatedMetricsTenant() { // enable tracking delayed metrics and track tracker.setIsTrackingDelayedMetrics(); List<String> delayedMetricNames = new ArrayList<String>() {{ for ( Metric metric : delayedMetrics ) { add(metric.getLocator().toString()); } }}; long ingestTime = System.currentTimeMillis(); tracker.trackDelayedAggregatedMetricsTenant(tenantId, delayedMetrics.get(0).getCollectionTime(), ingestTime, delayedMetricNames); // verify verify(loggerMock, atLeastOnce()).info("[TRACKER] Tracking delayed metrics started"); verify(loggerMock, atLeastOnce()).info("[TRACKER][DELAYED METRIC] Tenant sending delayed metrics " + tenantId); verify(loggerMock, atLeastOnce()).info(contains("[TRACKER][DELAYED METRIC] " + tenantId + ".delayed.metric1" + "," + tenantId + ".delayed.metric2 have collectionTime 2016-01-01 00:00:00 which is delayed")); }
public static Collection<PreaggregatedMetric> convertSets(String tenant, long timestamp, Collection<BluefloodSet> sets) { List<PreaggregatedMetric> list = new ArrayList<PreaggregatedMetric>(sets.size()); for (BluefloodSet set : sets) { Locator locator = Locator.createLocatorFromPathComponents(tenant, set.getName().split(NAME_DELIMITER, -1)); BluefloodSetRollup rollup = new BluefloodSetRollup(); for (String value : set.getValues()) { rollup = rollup.withObject(value); } PreaggregatedMetric metric = new PreaggregatedMetric(timestamp, locator, DEFAULT_TTL, rollup); list.add(metric); } return list; }
@Test public void testDeDupMetrics() throws Exception { // New index name and the locator to be written to it String ES_DUP = ElasticIO.ELASTICSEARCH_INDEX_NAME_WRITE + "_2"; Locator testLocator = createTestLocator(TENANT_A, 0, "A", 0); // Metric is already there in old List<SearchResult> results = elasticIO.search(TENANT_A, testLocator.getMetricName()); assertEquals(results.size(), 1); assertEquals(results.get(0).getMetricName(), testLocator.getMetricName()); // Insert metric into the new index elasticIO.setINDEX_NAME_WRITE(ES_DUP); List<IMetric> metricList = new ArrayList(); metricList.add(new Metric(createTestLocator(TENANT_A, 0, "A", 0), 987654321L, 0, new TimeValue(1, TimeUnit.DAYS), UNIT)); // Calling insertDiscovery with single metric in loop rather than metrics collection to improve on code coverage. for(IMetric metric : metricList){ elasticIO.insertDiscovery(metric); } helper.refreshIndex(ES_DUP); elasticIO.setINDEX_NAME_READ("metric_metadata_read"); results = elasticIO.search(TENANT_A, testLocator.getMetricName()); // Should just be one result assertEquals(results.size(), 1); assertEquals(results.get(0).getMetricName(), testLocator.getMetricName()); elasticIO.setINDEX_NAME_READ(ElasticIOConfig.ELASTICSEARCH_INDEX_NAME_READ.getDefaultValue()); elasticIO.setINDEX_NAME_WRITE(ElasticIOConfig.ELASTICSEARCH_INDEX_NAME_WRITE.getDefaultValue()); }
@Test public void testJSONMetricsContainerConstruction() throws Exception { // Construct the JSONMetricsContainter from JSON metric objects JSONMetricsContainer jsonMetricsContainer = getContainer( "ac1", generateJSONMetricsData() ); List<Metric> metricsCollection = jsonMetricsContainer.getValidMetrics(); assertTrue( jsonMetricsContainer.getValidationErrors().isEmpty() ); assertTrue( metricsCollection.size() == 2 ); assertEquals( "ac1.mzord.duration", metricsCollection.get( 0 ).getLocator().toString() ); assertEquals( Long.MAX_VALUE, metricsCollection.get( 0 ).getMetricValue() ); assertEquals( 1234566, metricsCollection.get( 0 ).getTtlInSeconds() ); assertTrue( current - metricsCollection.get( 0 ).getCollectionTime() < MINUTE ); assertEquals( "milliseconds", metricsCollection.get( 0 ).getUnit() ); assertEquals( "ac1.mzord.status", metricsCollection.get( 1 ).getLocator().toString() ); assertEquals( 0, metricsCollection.get( 1 ).getMetricValue() ); assertEquals( "unknown", metricsCollection.get( 1 ).getUnit() ); }
/** * * The below code generate locator's which match the below regex. * * one\.two\.three[0-2][0-9].four[A-C].five[0-2] * * Examples: * one.two.three00.fourA.five0 * one.two.three29.fourC.five2 * */ protected static Locator createTestLocator(String tenantId, int x, String y, int z) { String xs = (x < 10 ? "0" : "") + String.valueOf(x); return Locator.createLocatorFromPathComponents( tenantId, "one", "two", "three" + xs, "four" + y, "five" + String.valueOf(z)); }
public static Collection<PreaggregatedMetric> convertCounters(String tenant, long timestamp, long flushIntervalMillis, Collection<BluefloodCounter> counters) { List<PreaggregatedMetric> list = new ArrayList<PreaggregatedMetric>(counters.size()); for (BluefloodCounter counter : counters) { Locator locator = Locator.createLocatorFromPathComponents(tenant, counter.getName().split(NAME_DELIMITER, -1)); // flushIntervalMillis could be zero (if not specified in the statsD config). long sampleCount = flushIntervalMillis > 0 ? (long)(counter.getRate().doubleValue() * ((double)flushIntervalMillis/1000d)) : 1; Rollup rollup = new BluefloodCounterRollup() .withCount(resolveNumber(counter.getValue())) .withRate(counter.getRate().doubleValue()) .withSampleCount((int)sampleCount); PreaggregatedMetric metric = new PreaggregatedMetric(timestamp, locator, DEFAULT_TTL, rollup); list.add(metric); } return list; }
public static Collection<PreaggregatedMetric> convertGauges(String tenant, long timestamp, Collection<BluefloodGauge> gauges) { List<PreaggregatedMetric> list = new ArrayList<PreaggregatedMetric>(gauges.size()); for (BluefloodGauge gauge : gauges) { Locator locator = Locator.createLocatorFromPathComponents(tenant, gauge.getName().split(NAME_DELIMITER, -1)); Points<SimpleNumber> points = new Points<SimpleNumber>(); points.add(new Points.Point<SimpleNumber>(timestamp, new SimpleNumber(resolveNumber(gauge.getValue())))); try { Rollup rollup = BluefloodGaugeRollup.buildFromRawSamples(points); PreaggregatedMetric metric = new PreaggregatedMetric(timestamp, locator, DEFAULT_TTL, rollup); list.add(metric); } catch (IOException ex) { throw new IOError(ex); } } return list; }
@Test public void testCreateSingleRequest_WithNullMetricName() throws IOException { final String TENANT_A = "12345"; Locator locator = Locator.createLocatorFromPathComponents(TENANT_A); assertEquals(0, Token.getTokens(locator).size()); }
protected void createTestMetrics(String tenantId, Set<String> fullyQualifiedMetricNames) throws Exception { List<IMetric> metrics = new ArrayList<IMetric>(); for (String metricName: fullyQualifiedMetricNames) { metrics.add(new Metric(Locator.createLocatorFromPathComponents(tenantId, metricName), 5647382910L, 0, new TimeValue(1, TimeUnit.DAYS), UNIT)); } createTestMetrics(metrics); }
public static Collection<PreaggregatedMetric> convertTimers(String tenant, long timestamp, Collection<BluefloodTimer> timers) { List<PreaggregatedMetric> list = new ArrayList<PreaggregatedMetric>(timers.size()); for (BluefloodTimer timer : timers) { Locator locator = Locator.createLocatorFromPathComponents(tenant, timer.getName().split(NAME_DELIMITER, -1)); BluefloodTimerRollup rollup = new BluefloodTimerRollup() .withCount(timer.getCount().longValue()) .withSampleCount(1) .withAverage(resolveNumber(timer.getAvg() == null ? 0.0d : timer.getAvg())) .withMaxValue(resolveNumber(timer.getMax() == null ? 0.0d : timer.getMax())) .withMinValue(resolveNumber(timer.getMin() == null ? 0.0d : timer.getMin())) .withCountPS(timer.getRate() == null ? 0.0d : timer.getRate().doubleValue()) .withSum(timer.getSum() == null ? 0L : timer.getSum().doubleValue()) .withVariance(Math.pow(timer.getStd() == null ? 0.0d : timer.getStd().doubleValue(), 2d)); for (Map.Entry<String, Percentile> entry : timer.getPercentiles().entrySet()) { // throw away max and sum. if (entry.getValue().getAvg() != null) { rollup.setPercentile(entry.getKey(), resolveNumber(entry.getValue().getAvg())); } } PreaggregatedMetric metric = new PreaggregatedMetric(timestamp, locator, DEFAULT_TTL, rollup); list.add(metric); } return list; }