@Override public Map<String, String> parseToMap(String input) { final Map<String, Object> inner = delegate.parseToMap(input); final String k = Preconditions.checkNotNull( inner.get(key), "Key column [%s] missing data in line [%s]", key, input ).toString(); // Just in case is long final Object val = inner.get(value); if (val == null) { // Skip null or missing values, treat them as if there were no row at all. return ImmutableMap.of(); } return ImmutableMap.of(k, val.toString()); }
@Override public boolean processLine(String line) { if (lines == Integer.MAX_VALUE) { throw new ISE("Cannot read more than %,d lines", Integer.MAX_VALUE); } final Map<K, V> kvMap = parser.parseToMap(line); map.putAll(kvMap); lines++; entries += kvMap.size(); return true; }
@Nullable private Map<String, Object> parseString(@Nullable String inputString) { initializeParser(); return parser.parseToMap(inputString); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> flatten(final Blackhole blackhole) { Map<String, Object> parsed = nestedParser.parseToMap(nestedInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } nestedCounter = (nestedCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> baseline(final Blackhole blackhole) { Map<String, Object> parsed = flatParser.parseToMap(flatInputs.get(flatCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } flatCounter = (flatCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> jqflatten(final Blackhole blackhole) { Map<String, Object> parsed = jqParser.parseToMap(jqInputs.get(jqCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } jqCounter = (jqCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> preflattenNestedParser(final Blackhole blackhole) { Map<String, Object> parsed = fieldDiscoveryParser.parseToMap(flatInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } nestedCounter = (nestedCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> forcedRootPaths(final Blackhole blackhole) { Map<String, Object> parsed = forcedPathParser.parseToMap(flatInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } nestedCounter = (nestedCounter + 1) % numEvents; return parsed; }
@Test(expected = ParseException.class) public void testFailure() { final String pattern = "AAAAA"; final List<String> fieldNames = Collections.singletonList( "dummy" ); final Parser<String, Object> parser = new RegexParser( pattern, Optional.of("@"), fieldNames ); String data = "BBBB"; parser.parseToMap(data); } }
@Test public void testSimple() { List<JSONPathFieldSpec> fields = new ArrayList<>(); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); final Map<String, Object> jsonMap = jsonParser.parseToMap(json); Assert.assertEquals( "jsonMap", ImmutableMap.of("one", "foo", "two", ImmutableList.of("bar", "baz"), "three", "qux"), jsonMap ); }
@Test public void testWithWhackyCharacters() { List<JSONPathFieldSpec> fields = new ArrayList<>(); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); final Map<String, Object> jsonMap = jsonParser.parseToMap(whackyCharacterJson); Assert.assertEquals( "jsonMap", ImmutableMap.of("one", "foo?"), jsonMap ); }
@Test public void testWithoutHeader() { final Parser<String, Object> parser = parserFactory.get(format); final String body = concat(format, "hello", "world", "foo"); final Map<String, Object> jsonMap = parser.parseToMap(body); Assert.assertEquals( "jsonMap", ImmutableMap.of("column_1", "hello", "column_2", "world", "column_3", "foo"), jsonMap ); }
@Test public void testWithNumbers() { List<JSONPathFieldSpec> fields = new ArrayList<>(); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); final Map<String, Object> jsonMap = jsonParser.parseToMap(numbersJson); Assert.assertEquals( "jsonMap", ImmutableMap.of("five", 5.0, "six", 6L, "many", 1234567878900L, "toomany", 1.23456789E21), jsonMap ); }
@Test public void testWithHeaderRow() { final Parser<String, Object> parser = parserFactory.get(format, true, 0); parser.startFileFromBeginning(); final String[] body = new String[]{ concat(format, "time", "value1", "value2"), concat(format, "hello", "world", "foo") }; Assert.assertNull(parser.parseToMap(body[0])); final Map<String, Object> jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), jsonMap ); }
@Test public void testWithHeader() { final String header = concat(format, "time", "value1", "value2"); final Parser<String, Object> parser = parserFactory.get(format, header); final String body = concat(format, "hello", "world", "foo"); final Map<String, Object> jsonMap = parser.parseToMap(body); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), jsonMap ); }
@Test public void testRejectDuplicates2() { List<JSONPathFieldSpec> fields = new ArrayList<>(); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "met-array", ".met.a")); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot have duplicate field definition: met-array"); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); jsonParser.parseToMap(nestedJson); }
@Test public void testRejectDuplicates() { List<JSONPathFieldSpec> fields = new ArrayList<>(); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot have duplicate field definition: met-array"); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); jsonParser.parseToMap(nestedJson); }
@Test public void testParseFail() { List<JSONPathFieldSpec> fields = new ArrayList<>(); thrown.expect(ParseException.class); thrown.expectMessage("Unable to parse row [" + notJson + "]"); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); jsonParser.parseToMap(notJson); } }
@Test public void testMakeParser() { final JavaScriptConfig config = JavaScriptConfig.getEnabledInstance(); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "function(str) { var parts = str.split(\"-\"); return { one: parts[0], two: parts[1] } }", config ); final Parser<String, Object> parser = spec.makeParser(); final Map<String, Object> obj = parser.parseToMap("x-y"); Assert.assertEquals(ImmutableMap.of("one", "x", "two", "y"), obj); }
@Test public void testWithoutStartFileFromBeginning() { expectedException.expect(UnsupportedOperationException.class); expectedException.expectMessage( "hasHeaderRow or maxSkipHeaderRows is not supported. Please check the indexTask supports these options." ); final int skipHeaderRows = 2; final Parser<String, Object> parser = parserFactory.get(format, false, skipHeaderRows); final String[] body = new String[]{ concat(format, "header", "line", "1"), concat(format, "header", "line", "2"), concat(format, "hello", "world", "foo") }; parser.parseToMap(body[0]); }