TimestampAggregatorFactory( String name, String fieldName, String timeFormat, Comparator<Long> comparator, Long initValue ) { this.name = name; this.fieldName = fieldName; this.timeFormat = timeFormat; this.comparator = comparator; this.initValue = initValue; this.timestampSpec = new TimestampSpec(fieldName, timeFormat, null); }
@JsonCreator public AvroParseSpec( @JsonProperty("timestampSpec") TimestampSpec timestampSpec, @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec, @JsonProperty("flattenSpec") JSONPathSpec flattenSpec ) { super( timestampSpec != null ? timestampSpec : new TimestampSpec(null, null, null), dimensionsSpec != null ? dimensionsSpec : DimensionsSpec.EMPTY, flattenSpec != null ? flattenSpec : JSONPathSpec.DEFAULT ); }
@JsonCreator public TimeAndDimsParseSpec( @JsonProperty("timestampSpec") TimestampSpec timestampSpec, @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec ) { super( timestampSpec != null ? timestampSpec : new TimestampSpec(null, null, null), dimensionsSpec != null ? dimensionsSpec : new DimensionsSpec(null, null, null) ); }
@JsonCreator public InfluxParseSpec( @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec, @JsonProperty("whitelistMeasurements") List<String> measurementWhitelist ) { super( new TimestampSpec(InfluxParser.TIMESTAMP_KEY, "millis", null), dimensionsSpec != null ? dimensionsSpec : new DimensionsSpec(null, null, null) ); this.measurementWhitelist = measurementWhitelist; }
public Parser getFlatParser() { JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), null, null ); return spec.makeParser(); }
public Parser getFieldDiscoveryParser() { List<JSONPathFieldSpec> fields = new ArrayList<>(); JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), flattenSpec, null ); return spec.makeParser(); }
@Test(expected = IllegalArgumentException.class) public void testColumnMissing() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new CSVParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")), new ArrayList<>(), new ArrayList<>() ), ",", Collections.singletonList("a"), false, 0 ); }
@Test public void testDimensionSpecRequired() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("parseSpec requires dimensionSpec"); @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), null, ",", null, Arrays.asList("a", "B"), false, 0 ); }
public Builder withTimestampSpec(InputRowParser parser) { if (parser != null && parser.getParseSpec() != null && parser.getParseSpec().getTimestampSpec() != null) { this.timestampSpec = parser.getParseSpec().getTimestampSpec(); } else { this.timestampSpec = new TimestampSpec(null, null, null); } return this; }
@Test public void testLowercasing() { JSONLowercaseParseSpec spec = new JSONLowercaseParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("A", "B")), new ArrayList<>(), new ArrayList<>() ) ); Parser parser = spec.makeParser(); Map<String, Object> event = parser.parseToMap("{\"timestamp\":\"2015-01-01\",\"A\":\"foo\"}"); Assert.assertEquals("foo", event.get("a")); } }
public static List<InputRow> inputRowsWithDimensions(final List<String> dimensions) { final MapInputRowParser parser = new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(dimensions), dimensions.isEmpty() ? ImmutableList.of("t", "c1", "c2") : null, null ) ) ); return ImmutableList.of( parser.parseBatch(ImmutableMap.of("t", "2000-01-01", "c1", "9", "c2", ImmutableList.of("a"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2000-01-02", "c1", "10.1", "c2", ImmutableList.of())).get(0), parser.parseBatch(ImmutableMap.of("t", "2000-01-03", "c1", "2", "c2", ImmutableList.of(""))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-01", "c1", "1", "c2", ImmutableList.of("a", "c"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-02", "c1", "4", "c2", ImmutableList.of("abc"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-03", "c1", "5")).get(0) ); }
public static IncrementalIndex loadIncrementalIndex( final IncrementalIndex retVal, final CharSource source ) throws IOException { final StringInputRowParser parser = new StringInputRowParser( new DelimitedParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(DIMENSION_SCHEMAS, null, null), "\t", "\u0001", Arrays.asList(COLUMNS), false, 0 ), "utf8" ); return loadIncrementalIndex(() -> retVal, source, parser); }
@Test public void testExtractTimestampWithMissingTimestampColumn() { TimestampSpec spec = new TimestampSpec(null, null, DateTimes.EPOCH); Assert.assertEquals( DateTimes.of("1970-01-01"), spec.extractTimestamp(ImmutableMap.of("dim", "foo")) ); }
@Test public void testExtractTimestamp() { TimestampSpec spec = new TimestampSpec("TIMEstamp", "yyyy-MM-dd", null); Assert.assertEquals( DateTimes.of("2014-03-01"), spec.extractTimestamp(ImmutableMap.of("TIMEstamp", "2014-03-01")) ); }
@Test public void testMakeParser() { final JavaScriptConfig config = JavaScriptConfig.getEnabledInstance(); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "function(str) { var parts = str.split(\"-\"); return { one: parts[0], two: parts[1] } }", config ); final Parser<String, Object> parser = spec.makeParser(); final Map<String, Object> obj = parser.parseToMap("x-y"); Assert.assertEquals(ImmutableMap.of("one", "x", "two", "y"), obj); }
@Test public void testMakeParserNotAllowed() { final JavaScriptConfig config = new JavaScriptConfig(false); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "abc", config ); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("JavaScript is disabled"); spec.makeParser(); } }
@Test public void testSerde() throws IOException { HashMap<String, Boolean> feature = new HashMap<String, Boolean>(); feature.put("ALLOW_UNQUOTED_CONTROL_CHARS", true); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")), null, null), null, feature ); final JSONParseSpec serde = (JSONParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("timestamp", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals(Arrays.asList("bar", "foo"), serde.getDimensionsSpec().getDimensionNames()); Assert.assertEquals(feature, serde.getFeatureSpec()); } }
@Test public void testContextualTimestampList() { String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; String[] dates = new String[]{ "2000-01-01T05:00:00", "2000-01-01T05:00:01", "2000-01-01T05:00:01", "2000-01-01T05:00:02", "2000-01-01T05:00:03", }; TimestampSpec spec = new TimestampSpec("TIMEstamp", DATE_FORMAT, null); DateTimes.UtcFormatter formatter = DateTimes.wrapFormatter(ISODateTimeFormat.dateHourMinuteSecond()); for (String date : dates) { DateTime dateTime = spec.extractTimestamp(ImmutableMap.of("TIMEstamp", date)); DateTime expectedDateTime = formatter.parse(date); Assert.assertEquals(expectedDateTime, dateTime); } } }
@Test public void testSerde() throws IOException { RegexParseSpec spec = new RegexParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "\u0001", Collections.singletonList("abc"), "abc" ); final RegexParseSpec serde = (RegexParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals("abc", serde.getPattern()); Assert.assertEquals("\u0001", serde.getListDelimiter()); Assert.assertEquals(Collections.singletonList("abc"), serde.getDimensionsSpec().getDimensionNames()); } }
private static class TestFirehoseFactory implements FirehoseFactory<InputRowParser> { public TestFirehoseFactory() { } @Override @SuppressWarnings("unchecked") public Firehose connect(InputRowParser parser, File temporaryDirectory) throws ParseException { return new TestFirehose(parser); } }