@Override public Object extractValue(InputRow inputRow, String metricName) { return inputRow.getRaw(metricName); } };
@Override protected void innerMap( InputRow inputRow, Context context ) throws IOException, InterruptedException { final Map<String, Iterable<String>> dims = new HashMap<>(); for (final String dim : inputRow.getDimensions()) { dims.put(dim, inputRow.getDimension(dim)); } helper.emitDimValueCounts(context, DateTimes.utc(inputRow.getTimestampFromEpoch()), dims); } }
@Override public boolean processLine(String line) throws IOException { InputRow parsed = parser.parse(line); builder.append(parsed.getTimestamp()); for (String dimension : parsed.getDimensions()) { builder.append('\t'); builder.append(parsed.getRaw(dimension)); } logWriter.write(builder.toString()); builder.setLength(0); return true; }
/** * @param timeStamp rollup up timestamp to be used to create group key * @param inputRow input row * * @return groupKey for the given input row */ public static List<Object> toGroupKey(long timeStamp, InputRow inputRow) { final Map<String, Set<String>> dims = new TreeMap<>(); for (final String dim : inputRow.getDimensions()) { final Set<String> dimValues = ImmutableSortedSet.copyOf(inputRow.getDimension(dim)); if (dimValues.size() > 0) { dims.put(dim, dimValues); } } return ImmutableList.of(timeStamp, dims); }
@Test public void testTransformOverwriteField() { // Transforms are allowed to overwrite fields, and to refer to the fields they overwrite; double-check this. final TransformSpec transformSpec = new TransformSpec( null, ImmutableList.of( new ExpressionTransform("x", "concat(x,y)", TestExprMacroTable.INSTANCE) ) ); final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER); final InputRow row = parser.parseBatch(ROW1).get(0); Assert.assertNotNull(row); Assert.assertEquals(DateTimes.of("2000-01-01").getMillis(), row.getTimestampFromEpoch()); Assert.assertEquals(DateTimes.of("2000-01-01"), row.getTimestamp()); Assert.assertEquals(ImmutableList.of("f", "x", "y"), row.getDimensions()); Assert.assertEquals(ImmutableList.of("foobar"), row.getDimension("x")); Assert.assertEquals(3.0, row.getMetric("b").doubleValue(), 0); Assert.assertNull(row.getRaw("f")); }
if (row.getTimestampFromEpoch() < minTimestamp) { throw new IAE("Cannot add row[%s] because it is below the minTimestamp[%s]", row, DateTimes.utc(minTimestamp)); final List<String> rowDimensions = row.getDimensions(); try { dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent( row.getRaw(dimension), true ); if (row.getTimestamp() != null) { truncated = gran.bucketStart(row.getTimestamp()).getMillis();
@Override public VarianceAggregatorCollector extractValue(InputRow inputRow, String metricName) { Object rawValue = inputRow.getRaw(metricName); if (rawValue instanceof VarianceAggregatorCollector) { return (VarianceAggregatorCollector) rawValue; } VarianceAggregatorCollector collector = new VarianceAggregatorCollector(); List<String> dimValues = inputRow.getDimension(metricName); if (dimValues != null && dimValues.size() > 0) { for (String dimValue : dimValues) { float value = Float.parseFloat(dimValue); collector.add(value); } } return collector; } };
out.writeLong(row.getTimestampFromEpoch()); List<String> dimList = row.getDimensions(); typeHelper.serialize(out, row.getRaw(dim));
@Test public void simpleFirehoseReadingTest() throws IOException { Assert.assertEquals(MAX_SHARD_NUMBER.longValue(), segmentSet.size()); Integer rowcount = 0; try (final IngestSegmentFirehose firehose = (IngestSegmentFirehose) factory.connect(rowParser, null)) { while (firehose.hasMore()) { InputRow row = firehose.nextRow(); Assert.assertArrayEquals(new String[]{DIM_NAME}, row.getDimensions().toArray()); Assert.assertArrayEquals(new String[]{DIM_VALUE}, row.getDimension(DIM_NAME).toArray()); Assert.assertEquals(METRIC_LONG_VALUE.longValue(), row.getMetric(METRIC_LONG_NAME)); Assert.assertEquals(METRIC_FLOAT_VALUE, row.getMetric(METRIC_FLOAT_NAME).floatValue(), METRIC_FLOAT_VALUE * 0.0001); ++rowcount; } } Assert.assertEquals((int) MAX_SHARD_NUMBER * MAX_ROWS, (int) rowcount); }
@Override public DateTime getTimestamp() { return row.getTimestamp(); }
@Override public List<String> getDimension(String dimension) { List<String> retVal = spatialLookup.get(dimension); return (retVal == null) ? row.getDimension(dimension) : retVal; }
/** * Get the proper bucket for some input row. * * @param inputRow an InputRow * * @return the Bucket that this row belongs to */ public Optional<Bucket> getBucket(InputRow inputRow) { final Optional<Interval> timeBucket = schema.getDataSchema().getGranularitySpec().bucketInterval( DateTimes.utc(inputRow.getTimestampFromEpoch()) ); if (!timeBucket.isPresent()) { return Optional.absent(); } final DateTime bucketStart = timeBucket.get().getStart(); final ShardSpec actualSpec = shardSpecLookups.get(bucketStart.getMillis()) .getShardSpec( rollupGran.bucketStart(inputRow.getTimestamp()).getMillis(), inputRow ); final HadoopyShardSpec hadoopyShardSpec = hadoopShardSpecLookup.get(bucketStart.getMillis()).get(actualSpec); return Optional.of( new Bucket( hadoopyShardSpec.getShardNum(), bucketStart, actualSpec.getPartitionNum() ) ); }
ByteBuffer.wrap("{\"time\":\"2000-01-01\",\"dimA\":\"foo\"}".getBytes(StandardCharsets.UTF_8)) ).get(0); Assert.assertEquals(DateTimes.of("2000-01-01"), row1bb.getTimestamp()); Assert.assertEquals("foo", row1bb.getRaw("dimA")); Assert.assertEquals("foofoo", row1bb.getRaw("expr")); Assert.assertEquals(DateTimes.of("2000-01-01"), row1string.getTimestamp()); Assert.assertEquals("foo", row1string.getRaw("dimA")); Assert.assertEquals("foofoo", row1string.getRaw("expr"));
@Override public long getTimestampFromEpoch() { return row.getTimestampFromEpoch(); }
EasyMock.expect(row.getTimestampFromEpoch()).andReturn(0L); EasyMock.expect(row.getDimensions()).andReturn(new ArrayList<String>()); EasyMock.replay(row);
@Test public void testCombiningfirehose() throws IOException { List<InputRow> list1 = Arrays.asList(makeRow(1, 1), makeRow(2, 2)); List<InputRow> list2 = Arrays.asList(makeRow(3, 3), makeRow(4, 4), makeRow(5, 5)); FirehoseFactory combiningFactory = new CombiningFirehoseFactory( Arrays.asList( new ListFirehoseFactory(list1), new ListFirehoseFactory(list2) ) ); final Firehose firehose = combiningFactory.connect(null, null); for (int i = 1; i < 6; i++) { Assert.assertTrue(firehose.hasMore()); final InputRow inputRow = firehose.nextRow(); Assert.assertEquals(i, inputRow.getTimestampFromEpoch()); Assert.assertEquals(i, inputRow.getMetric("test").floatValue(), 0); } Assert.assertFalse(firehose.hasMore()); }
@Override public List<String> getDimensions() { return row.getDimensions(); }
@Override public Number getMetric(String metric) { return row.getMetric(metric); }
@Test public void testTransforms() { final TransformSpec transformSpec = new TransformSpec( null, ImmutableList.of( new ExpressionTransform("f", "concat(x,y)", TestExprMacroTable.INSTANCE), new ExpressionTransform("g", "a + b", TestExprMacroTable.INSTANCE), new ExpressionTransform("h", "concat(f,g)", TestExprMacroTable.INSTANCE) ) ); final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER); final InputRow row = parser.parseBatch(ROW1).get(0); Assert.assertNotNull(row); Assert.assertEquals(DateTimes.of("2000-01-01").getMillis(), row.getTimestampFromEpoch()); Assert.assertEquals(DateTimes.of("2000-01-01"), row.getTimestamp()); Assert.assertEquals(ImmutableList.of("f", "x", "y"), row.getDimensions()); Assert.assertEquals(ImmutableList.of("foo"), row.getDimension("x")); Assert.assertEquals(3.0, row.getMetric("b").doubleValue(), 0); Assert.assertEquals("foobar", row.getRaw("f")); Assert.assertEquals(ImmutableList.of("foobar"), row.getDimension("f")); Assert.assertEquals(ImmutableList.of("5.0"), row.getDimension("g")); Assert.assertEquals(ImmutableList.of(), row.getDimension("h")); Assert.assertEquals(5L, row.getMetric("g").longValue()); }
if (row.getTimestampFromEpoch() < minTimestamp) { throw new IAE("Cannot add row[%s] because it is below the minTimestamp[%s]", row, DateTimes.utc(minTimestamp)); final List<String> rowDimensions = row.getDimensions(); try { dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent( row.getRaw(dimension), true ); if (row.getTimestamp() != null) { truncated = gran.bucketStart(row.getTimestamp()).getMillis();