@Override public boolean processLine(String line) { if (lines == Integer.MAX_VALUE) { throw new ISE("Cannot read more than %,d lines", Integer.MAX_VALUE); } final Map<K, V> kvMap = parser.parseToMap(line); map.putAll(kvMap); lines++; entries += kvMap.size(); return true; }
public void startFileFromBeginning() { initializeParser(); parser.startFileFromBeginning(); }
@Override public List<String> getFieldNames() { return delegate.getFieldNames(); } }
@Test public void testWithHeaderRow() { final Parser<String, Object> parser = parserFactory.get(format, true, 0); parser.startFileFromBeginning(); final String[] body = new String[]{ concat(format, "time", "value1", "value2"), concat(format, "hello", "world", "foo") }; Assert.assertNull(parser.parseToMap(body[0])); final Map<String, Object> jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), jsonMap ); }
@Override public void setFieldNames(Iterable<String> fieldNames) { delegate.setFieldNames(fieldNames); }
@Test public void testWithHeaderRowOfEmptyColumns() { final Parser<String, Object> parser = parserFactory.get(format, true, 0); parser.startFileFromBeginning(); final String[] body = new String[]{ concat(format, "time", "", "value2", ""), concat(format, "hello", "world", "foo", "bar") }; Assert.assertNull(parser.parseToMap(body[0])); final Map<String, Object> jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "column_2", "world", "value2", "foo", "column_4", "bar"), jsonMap ); }
@Nullable private Map<String, Object> parseString(@Nullable String inputString) { initializeParser(); return parser.parseToMap(inputString); }
@Test public void testWithDifferentHeaderRows() { final Parser<String, Object> parser = parserFactory.get(format, true, 0); parser.startFileFromBeginning(); final String[] body = new String[]{ concat(format, "time", "value1", "value2"), concat(format, "hello", "world", "foo") }; Assert.assertNull(parser.parseToMap(body[0])); Map<String, Object> jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), jsonMap ); parser.startFileFromBeginning(); final String[] body2 = new String[]{ concat(format, "time", "value1", "value2", "value3"), concat(format, "hello", "world", "foo", "bar") }; Assert.assertNull(parser.parseToMap(body2[0])); jsonMap = parser.parseToMap(body2[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo", "value3", "bar"), jsonMap ); }
public void startFileFromBeginning() { initializeParser(); parser.startFileFromBeginning(); }
@Test public void testValidHeader() { final String header = concat(format, "time", "value1", "value2"); final Parser<String, Object> parser = parserFactory.get(format, header); Assert.assertEquals(ImmutableList.of("time", "value1", "value2"), parser.getFieldNames()); }
@Override public Map<String, String> parseToMap(String input) { final Map<String, Object> inner = delegate.parseToMap(input); final String k = Preconditions.checkNotNull( inner.get(key), "Key column [%s] missing data in line [%s]", key, input ).toString(); // Just in case is long final Object val = inner.get(value); if (val == null) { // Skip null or missing values, treat them as if there were no row at all. return ImmutableMap.of(); } return ImmutableMap.of(k, val.toString()); }
@Test public void testWithSkipHeaderRows() { final int skipHeaderRows = 2; final Parser<String, Object> parser = parserFactory.get(format, false, skipHeaderRows); parser.startFileFromBeginning(); final String[] body = new String[]{ concat(format, "header", "line", "1"), concat(format, "header", "line", "2"), concat(format, "hello", "world", "foo") }; int index; for (index = 0; index < skipHeaderRows; index++) { Assert.assertNull(parser.parseToMap(body[index])); } final Map<String, Object> jsonMap = parser.parseToMap(body[index]); Assert.assertEquals( "jsonMap", ImmutableMap.of("column_1", "hello", "column_2", "world", "column_3", "foo"), jsonMap ); }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> flatten(final Blackhole blackhole) { Map<String, Object> parsed = nestedParser.parseToMap(nestedInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } nestedCounter = (nestedCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> baseline(final Blackhole blackhole) { Map<String, Object> parsed = flatParser.parseToMap(flatInputs.get(flatCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } flatCounter = (flatCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> jqflatten(final Blackhole blackhole) { Map<String, Object> parsed = jqParser.parseToMap(jqInputs.get(jqCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } jqCounter = (jqCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> preflattenNestedParser(final Blackhole blackhole) { Map<String, Object> parsed = fieldDiscoveryParser.parseToMap(flatInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } nestedCounter = (nestedCounter + 1) % numEvents; return parsed; }
@Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map<String, Object> forcedRootPaths(final Blackhole blackhole) { Map<String, Object> parsed = forcedPathParser.parseToMap(flatInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } nestedCounter = (nestedCounter + 1) % numEvents; return parsed; }
@Test public void testParseWithMultiVal() { final String function = "function(str) { var parts = str.split(\"-\"); return { one: [parts[0], parts[1]] } }"; final Parser<String, Object> parser = new JavaScriptParser( function ); String data = "val1-val2"; final Map<String, Object> parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("one", Lists.newArrayList("val1", "val2")); Assert.assertEquals( "result", builder.build(), parsed ); }
@Test public void testParse() { final String function = "function(str) { var parts = str.split(\"-\"); return { one: parts[0], two: parts[1] } }"; final Parser<String, Object> parser = new JavaScriptParser( function ); String data = "foo-val1"; final Map<String, Object> parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("one", "foo"); builder.put("two", "val1"); Assert.assertEquals( "result", builder.build(), parsed ); }
@Test public void testMultiValWithRegexSplit() { final String pattern = "(.*)"; final String listPattern = "[a-f]"; final Parser<String, Object> parser = new RegexParser( pattern, Optional.of(listPattern) ); String data = "1a2"; final Map<String, Object> parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("column_1", Lists.newArrayList("1", "2")); Assert.assertEquals( "result", builder.build(), parsed ); }