public SchemaBuilder addMetric(@Nonnull String name, @Nonnull DataType dataType, int fieldSize, @Nonnull MetricFieldSpec.DerivedMetricType derivedMetricType) { _schema.addField(new MetricFieldSpec(name, dataType, fieldSize, derivedMetricType)); return this; }
@Override public GenericRow aggregateRecords(List<GenericRow> rows) { GenericRow resultRow = rows.get(0); for (int i = 1; i < rows.size(); i++) { GenericRow currentRow = rows.get(i); for (MetricFieldSpec metric : _schema.getMetricFieldSpecs()) { String metricName = metric.getName(); ValueAggregator aggregator = (_valueAggregatorMap == null) ? DEFAULT_AGGREGATOR_FUNCTION : _valueAggregatorMap.getOrDefault(metricName, DEFAULT_AGGREGATOR_FUNCTION); Object aggregatedResult = aggregator.aggregate(resultRow.getValue(metricName), currentRow.getValue(metricName), metric); resultRow.putField(metricName, aggregatedResult); } } return resultRow; } }
/** * Test derived {@link MetricFieldSpec}. */ @Test public void testDerivedMetricFieldSpec() throws Exception { MetricFieldSpec derivedMetricField = new MetricFieldSpec("derivedMetric", STRING, 10, MetricFieldSpec.DerivedMetricType.HLL); Assert.assertEquals(derivedMetricField.getFieldSize(), 10); Assert.assertTrue(derivedMetricField.isDerivedMetric()); Assert.assertEquals(derivedMetricField.getDerivedMetricType(), MetricFieldSpec.DerivedMetricType.HLL); Assert.assertEquals(derivedMetricField.getDefaultNullValue(), "null"); // Test serialize deserialize. MetricFieldSpec derivedMetricField2 = JsonUtils.stringToObject(derivedMetricField.toJsonObject().toString(), MetricFieldSpec.class); Assert.assertEquals(derivedMetricField2, derivedMetricField); }
public Object getValueConformToDataType(int index) { MetricFieldSpec metricFieldSpec = _metricFieldSpecs.get(index); if (metricFieldSpec.getDataType() == FieldSpec.DataType.STRING) { assert metricFieldSpec.getDerivedMetricType() == DerivedMetricType.HLL; return HllUtil.convertHllToString((HyperLogLog) _values[index]); } else { return _values[index]; } } }
for (int i = 0; i < numMetrics; i++) { MetricFieldSpec metricFieldSpec = metricFieldSpecs.get(i); switch (metricFieldSpec.getDataType()) { case INT: values[i] = buffer.getInt(); break; case STRING: assert metricFieldSpec.getDerivedMetricType() == DerivedMetricType.HLL; byte[] hllBytes = new byte[metricFieldSpec.getFieldSize()]; buffer.get(hllBytes); values[i] = HllUtil.buildHllFromBytes(hllBytes);
for (int i = 0; i < NUM_METRICS; i++) { String metricName = "m" + (i + 1); MetricFieldSpec metricFieldSpec = new MetricFieldSpec(metricName, FieldSpec.DataType.INT); schema.addField(metricFieldSpec); String metName = schema.getMetricFieldSpecs().get(i).getName(); map.put(metName, RANDOM.nextInt(METRIC_MAX_VALUE));
Assert.assertEquals(metricFieldSpec.getFieldType(), FieldSpec.FieldType.METRIC); Assert.assertEquals(metricFieldSpec.getName(), "metric"); Assert.assertEquals(metricFieldSpec.getDataType(), FieldSpec.DataType.INT); Assert.assertEquals(metricFieldSpec.isSingleValueField(), true); Assert.assertEquals(metricFieldSpec.getDefaultNullValue(), 0); Assert.assertEquals(metricFieldSpec.getFieldSize(), 4); Assert.assertNull(metricFieldSpec.getDerivedMetricType()); Assert.assertEquals(metricFieldSpec.getFieldType(), FieldSpec.FieldType.METRIC); Assert.assertEquals(metricFieldSpec.getName(), "metricWithDefault"); Assert.assertEquals(metricFieldSpec.getDataType(), FieldSpec.DataType.INT); Assert.assertEquals(metricFieldSpec.isSingleValueField(), true); Assert.assertEquals(metricFieldSpec.getDefaultNullValue(), 5); Assert.assertEquals(metricFieldSpec.getFieldSize(), 4); Assert.assertNull(metricFieldSpec.getDerivedMetricType()); Assert.assertEquals(metricFieldSpec.getFieldType(), FieldSpec.FieldType.METRIC); Assert.assertEquals(metricFieldSpec.getName(), "derivedMetric"); Assert.assertEquals(metricFieldSpec.getDataType(), FieldSpec.DataType.STRING); Assert.assertEquals(metricFieldSpec.isSingleValueField(), true); Assert.assertEquals(metricFieldSpec.getDefaultNullValue(), "null"); Assert.assertEquals(metricFieldSpec.getFieldSize(), 10); Assert.assertNotNull(metricFieldSpec.getDerivedMetricType()); Assert.assertEquals(metricFieldSpec.getFieldType(), FieldSpec.FieldType.METRIC); Assert.assertEquals(metricFieldSpec.getName(), "derivedMetricWithDefault"); Assert.assertEquals(metricFieldSpec.getDataType(), FieldSpec.DataType.STRING); Assert.assertEquals(metricFieldSpec.isSingleValueField(), true); Assert.assertEquals(metricFieldSpec.getDefaultNullValue(), defaultString);
String column = metricFieldSpec.getName(); if (metricFieldSpec.isDerivedMetric()) { assert metricFieldSpec.getDerivedMetricType() == MetricFieldSpec.DerivedMetricType.HLL; functionColumnPairs.add(new AggregationFunctionColumnPair(AggregationFunctionType.FASTHLL, column)); } else {
MetricFieldSpec metricFieldSpec1 = JsonUtils.stringToObject(getRandomOrderJsonString(metricFields), MetricFieldSpec.class); MetricFieldSpec metricFieldSpec2 = new MetricFieldSpec("metric", INT, -1); Assert.assertEquals(metricFieldSpec1, metricFieldSpec2, ERROR_MESSAGE); Assert.assertEquals(metricFieldSpec1.getDefaultNullValue(), -1, ERROR_MESSAGE);
_metricNames.add(metricFieldSpec.getName()); _metricSize += metricFieldSpec.getFieldSize();
private static void printSchema(Schema schema) { LOGGER.info("schemaName: {}", schema.getSchemaName()); LOGGER.info("Dimension columnNames: "); int i = 0; for (DimensionFieldSpec spec : schema.getDimensionFieldSpecs()) { String columnInfo = i + " " + spec.getName(); if (!spec.isSingleValueField()) { LOGGER.info(columnInfo + " Multi-Value."); } else { LOGGER.info(columnInfo); } i += 1; } LOGGER.info("Metric columnNames: "); i = 0; for (MetricFieldSpec spec : schema.getMetricFieldSpecs()) { String columnInfo = i + " " + spec.getName(); if (!spec.isSingleValueField()) { LOGGER.info(columnInfo + " Multi-Value."); } else { LOGGER.info(columnInfo); } i += 1; } LOGGER.info("Time column: {}", schema.getTimeColumnName()); }
case STRING: MetricFieldSpec metricFieldSpec = (MetricFieldSpec) fieldSpec; if (!metricFieldSpec.isDerivedMetric()) { ctxLogger.info("Unsupported data type: STRING in non-derived METRIC field: {}", fieldName); return false;
@Override public Object aggregate(Object value1, Object value2, MetricFieldSpec metricFieldSpec) { Object result; switch (metricFieldSpec.getDataType()) { case LONG: result = ((Number) value1).longValue() + ((Number) value2).longValue(); break; case FLOAT: result = ((Number) value1).floatValue() + ((Number) value2).floatValue(); break; case DOUBLE: result = ((Number) value1).doubleValue() + ((Number) value2).doubleValue(); break; default: throw new IllegalArgumentException( "Unsupported metric type for SUM aggregator : " + metricFieldSpec.getDataType()); } return result; } }
ArrayNode jsonArray = JsonUtils.newArrayNode(); for (MetricFieldSpec metricFieldSpec : _metricFieldSpecs) { jsonArray.add(metricFieldSpec.toJsonObject());
String metricName = _metricNames.get(i); Object metricValue = row.getValue(metricName); if (_metricFieldSpecs.get(i).getDerivedMetricType() == MetricFieldSpec.DerivedMetricType.HLL) {
for (int i = 0; i < numMetrics; i++) { String metricName = "m" + (i + 1); MetricFieldSpec metricFieldSpec = new MetricFieldSpec(metricName, DataType.INT); schema.addField(metricFieldSpec); String metName = schema.getMetricFieldSpecs().get(i).getName(); map.put(metName, 1);
public void aggregate(MetricBuffer buffer) { int numValues = _values.length; for (int i = 0; i < numValues; i++) { MetricFieldSpec metricFieldSpec = _metricFieldSpecs.get(i); switch (metricFieldSpec.getDataType()) { case INT: _values[i] = (Integer) _values[i] + (Integer) buffer._values[i]; break; case LONG: _values[i] = (Long) _values[i] + (Long) buffer._values[i]; break; case FLOAT: _values[i] = (Float) _values[i] + (Float) buffer._values[i]; break; case DOUBLE: _values[i] = (Double) _values[i] + (Double) buffer._values[i]; break; case STRING: assert metricFieldSpec.getDerivedMetricType() == DerivedMetricType.HLL; try { ((HyperLogLog) _values[i]).addAll((HyperLogLog) buffer._values[i]); } catch (CardinalityMergeException e) { throw new RuntimeException(e); } break; default: throw new IllegalStateException(); } } }
@Override public Object aggregate(Object value1, Object value2, MetricFieldSpec metricFieldSpec) { Object result; switch (metricFieldSpec.getDataType()) { case INT: result = Math.max(((Number) value1).intValue(), ((Number) value2).intValue()); break; case LONG: result = Math.max(((Number) value1).longValue(), ((Number) value2).longValue()); break; case FLOAT: result = Math.max(((Number) value1).floatValue(), ((Number) value2).floatValue()); break; case DOUBLE: result = Math.max(((Number) value1).doubleValue(), ((Number) value2).doubleValue()); break; default: throw new IllegalArgumentException("Unsupported metric type : " + metricFieldSpec.getDataType()); } return result; } }
public SchemaBuilder addMetric(@Nonnull String name, @Nonnull DataType dataType, int fieldSize, @Nonnull MetricFieldSpec.DerivedMetricType derivedMetricType, @Nonnull Object defaultNullValue) { _schema.addField(new MetricFieldSpec(name, dataType, fieldSize, derivedMetricType, defaultNullValue)); return this; }
MetricFieldSpec metricFieldSpec = new MetricFieldSpec(metricName, DataType.INT); schema.addField(metricFieldSpec); new MetricFieldSpec(hllMetricName, FieldSpec.DataType.STRING, HllSizeUtils.getHllFieldSizeFromLog2m(log2m), MetricFieldSpec.DerivedMetricType.HLL); schema.addField(hllDerivedFieldSpec); String metName = schema.getMetricFieldSpecs().get(i).getName(); map.put(metName, 1);