public static JSONPathFieldSpec createNestedField(String name, String expr) { return new JSONPathFieldSpec(JSONPathFieldType.PATH, name, expr); }
fields.add(JSONPathFieldSpec.createRootField("ts")); fields.add(JSONPathFieldSpec.createRootField("d1")); fields.add(JSONPathFieldSpec.createNestedField("e1.d1", "$.e1.d1")); fields.add(JSONPathFieldSpec.createNestedField("e1.d2", "$.e1.d2")); fields.add(JSONPathFieldSpec.createNestedField("e2.d3", "$.e2.d3")); fields.add(JSONPathFieldSpec.createNestedField("e2.d4", "$.e2.d4")); fields.add(JSONPathFieldSpec.createNestedField("e2.d5", "$.e2.d5")); fields.add(JSONPathFieldSpec.createNestedField("e2.d6", "$.e2.d6")); fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[0]", "$.e2.ad1[0]")); fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[1]", "$.e2.ad1[1]")); fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[2]", "$.e2.ad1[2]")); fields.add(JSONPathFieldSpec.createNestedField("ae1[0].d1", "$.ae1[0].d1")); fields.add(JSONPathFieldSpec.createNestedField("ae1[1].d1", "$.ae1[1].d1")); fields.add(JSONPathFieldSpec.createNestedField("ae1[2].e1.d2", "$.ae1[2].e1.d2")); fields.add(JSONPathFieldSpec.createRootField("m3")); fields.add(JSONPathFieldSpec.createNestedField("e3.m1", "$.e3.m1")); fields.add(JSONPathFieldSpec.createNestedField("e3.m2", "$.e3.m2")); fields.add(JSONPathFieldSpec.createNestedField("e3.m3", "$.e3.m3")); fields.add(JSONPathFieldSpec.createNestedField("e3.m4", "$.e3.m4")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[0]", "$.e3.am1[0]")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[1]", "$.e3.am1[1]")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[2]", "$.e3.am1[2]")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[3]", "$.e3.am1[3]")); fields.add(JSONPathFieldSpec.createNestedField("e4.e4.m4", "$.e4.e4.m4"));
public Parser getJqParser() fields.add(JSONPathFieldSpec.createRootField("ts")); fields.add(JSONPathFieldSpec.createRootField("d1")); fields.add(JSONPathFieldSpec.createJqField("e1.d1", ".e1.d1")); fields.add(JSONPathFieldSpec.createJqField("e1.d2", ".e1.d2")); fields.add(JSONPathFieldSpec.createJqField("e2.d3", ".e2.d3")); fields.add(JSONPathFieldSpec.createJqField("e2.d4", ".e2.d4")); fields.add(JSONPathFieldSpec.createJqField("e2.d5", ".e2.d5")); fields.add(JSONPathFieldSpec.createJqField("e2.d6", ".e2.d6")); fields.add(JSONPathFieldSpec.createJqField("e2.ad1[0]", ".e2.ad1[0]")); fields.add(JSONPathFieldSpec.createJqField("e2.ad1[1]", ".e2.ad1[1]")); fields.add(JSONPathFieldSpec.createJqField("e2.ad1[2]", ".e2.ad1[2]")); fields.add(JSONPathFieldSpec.createJqField("ae1[0].d1", ".ae1[0].d1")); fields.add(JSONPathFieldSpec.createJqField("ae1[1].d1", ".ae1[1].d1")); fields.add(JSONPathFieldSpec.createJqField("ae1[2].e1.d2", ".ae1[2].e1.d2")); fields.add(JSONPathFieldSpec.createRootField("m3")); fields.add(JSONPathFieldSpec.createJqField("e3.m1", ".e3.m1")); fields.add(JSONPathFieldSpec.createJqField("e3.m2", ".e3.m2")); fields.add(JSONPathFieldSpec.createJqField("e3.m3", ".e3.m3")); fields.add(JSONPathFieldSpec.createJqField("e3.m4", ".e3.m4")); fields.add(JSONPathFieldSpec.createJqField("e3.am1[0]", ".e3.am1[0]")); fields.add(JSONPathFieldSpec.createJqField("e3.am1[1]", ".e3.am1[1]")); fields.add(JSONPathFieldSpec.createJqField("e3.am1[2]", ".e3.am1[2]")); fields.add(JSONPathFieldSpec.createJqField("e3.am1[3]", ".e3.am1[3]")); fields.add(JSONPathFieldSpec.createJqField("e4.e4.m4", ".e4.e4.m4"));
fields.add(JSONPathFieldSpec.createNestedField("foobar1", "$.foo.bar1")); fields.add(JSONPathFieldSpec.createNestedField("baz0", "$.baz[0]")); fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx")); fields.add(JSONPathFieldSpec.createRootField("timestamp")); fields.add(JSONPathFieldSpec.createRootField("foo.bar1")); fields.add(JSONPathFieldSpec.createJqField("foobar1", ".foo.bar1")); fields.add(JSONPathFieldSpec.createJqField("baz0", ".baz[0]")); fields.add(JSONPathFieldSpec.createJqField("hey0barx", ".hey[0].barx")); Assert.assertEquals(JSONPathFieldType.PATH, foobar1.getType()); Assert.assertEquals("foobar1", foobar1.getName()); Assert.assertEquals("$.foo.bar1", foobar1.getExpr()); Assert.assertEquals(JSONPathFieldType.PATH, baz0.getType()); Assert.assertEquals("baz0", baz0.getName()); Assert.assertEquals("$.baz[0]", baz0.getExpr()); Assert.assertEquals(JSONPathFieldType.PATH, hey0barx.getType()); Assert.assertEquals("hey0barx", hey0barx.getName()); Assert.assertEquals("$.hey[0].barx", hey0barx.getExpr()); Assert.assertEquals(JSONPathFieldType.JQ, jqFoobar1.getType()); Assert.assertEquals("foobar1", jqFoobar1.getName()); Assert.assertEquals(".foo.bar1", jqFoobar1.getExpr()); Assert.assertEquals(JSONPathFieldType.JQ, jqBaz0.getType()); Assert.assertEquals("baz0", jqBaz0.getName()); Assert.assertEquals(".baz[0]", jqBaz0.getExpr());
final Function<T, Object> extractor; switch (fieldSpec.getType()) { case ROOT: extractor = obj -> flattenerMaker.getRootField(obj, fieldSpec.getExpr()); break; case PATH: extractor = flattenerMaker.makeJsonPathExtractor(fieldSpec.getExpr()); break; case JQ: extractor = flattenerMaker.makeJsonQueryExtractor(fieldSpec.getExpr()); break; default: throw new UOE("Unsupported field type[%s]", fieldSpec.getType()); if (extractors.put(fieldSpec.getName(), extractor) != null) { throw new IAE("Cannot have duplicate field definition: %s", fieldSpec.getName());
@JsonCreator public static JSONPathFieldSpec fromString(String name) { return JSONPathFieldSpec.createRootField(name); }
fields.add(JSONPathFieldSpec.createNestedField("ts", "$['ts']")); fields.add(JSONPathFieldSpec.createNestedField("d1", "$['d1']")); fields.add(JSONPathFieldSpec.createNestedField("d2", "$['d2']")); fields.add(JSONPathFieldSpec.createNestedField("e1.d1", "$['e1.d1']")); fields.add(JSONPathFieldSpec.createNestedField("e1.d2", "$['e1.d2']")); fields.add(JSONPathFieldSpec.createNestedField("e2.d3", "$['e2.d3']")); fields.add(JSONPathFieldSpec.createNestedField("e2.d4", "$['e2.d4']")); fields.add(JSONPathFieldSpec.createNestedField("e2.d5", "$['e2.d5']")); fields.add(JSONPathFieldSpec.createNestedField("e2.d6", "$['e2.d6']")); fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[0]", "$['e2.ad1[0]']")); fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[1]", "$['e2.ad1[1]']")); fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[2]", "$['e2.ad1[2]']")); fields.add(JSONPathFieldSpec.createNestedField("ae1[0].d1", "$['ae1[0].d1']")); fields.add(JSONPathFieldSpec.createNestedField("ae1[1].d1", "$['ae1[1].d1']")); fields.add(JSONPathFieldSpec.createNestedField("ae1[2].e1.d2", "$['ae1[2].e1.d2']")); fields.add(JSONPathFieldSpec.createNestedField("m3", "$['m3']")); fields.add(JSONPathFieldSpec.createNestedField("m4", "$['m4']")); fields.add(JSONPathFieldSpec.createNestedField("e3.m1", "$['e3.m1']")); fields.add(JSONPathFieldSpec.createNestedField("e3.m2", "$['e3.m2']")); fields.add(JSONPathFieldSpec.createNestedField("e3.m3", "$['e3.m3']")); fields.add(JSONPathFieldSpec.createNestedField("e3.m4", "$['e3.m4']")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[0]", "$['e3.am1[0]']")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[1]", "$['e3.am1[1]']")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[2]", "$['e3.am1[2]']")); fields.add(JSONPathFieldSpec.createNestedField("e3.am1[3]", "$['e3.am1[3]']")); fields.add(JSONPathFieldSpec.createNestedField("e4.e4.m4", "$['e4.e4.m4']"));
fields.add(JSONPathFieldSpec.createNestedField("foobar1", "$.foo.bar1")); fields.add(JSONPathFieldSpec.createNestedField("foobar2", "$.foo.bar2")); fields.add(JSONPathFieldSpec.createNestedField("baz0", "$.baz[0]")); fields.add(JSONPathFieldSpec.createNestedField("baz1", "$.baz[1]")); fields.add(JSONPathFieldSpec.createNestedField("baz2", "$.baz[2]")); fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx")); fields.add(JSONPathFieldSpec.createNestedField("metA", "$.met.a")); fields.add(JSONPathFieldSpec.createNestedField("missing", "$.nonexistent.nested.field")); fields.add(JSONPathFieldSpec.createRootField("timestamp")); fields.add(JSONPathFieldSpec.createRootField("foo.bar1")); final JSONPathSpec parsedSpec = jsonMapper.readValue(testSpec, JSONPathSpec.class); List<JSONPathFieldSpec> fieldSpecs = parsedSpec.getFields(); Assert.assertEquals(JSONPathFieldType.ROOT, fieldSpecs.get(0).getType()); Assert.assertEquals("parseThisRootField", fieldSpecs.get(0).getName()); Assert.assertEquals("parseThisRootField", fieldSpecs.get(0).getExpr());
final Function<T, Object> extractor; switch (fieldSpec.getType()) { case ROOT: extractor = obj -> flattenerMaker.getRootField(obj, fieldSpec.getExpr()); break; case PATH: extractor = flattenerMaker.makeJsonPathExtractor(fieldSpec.getExpr()); break; case JQ: extractor = flattenerMaker.makeJsonQueryExtractor(fieldSpec.getExpr()); break; default: throw new UOE("Unsupported field type[%s]", fieldSpec.getType()); if (extractors.put(fieldSpec.getName(), extractor) != null) { throw new IAE("Cannot have duplicate field definition: %s", fieldSpec.getName());
@JsonCreator public static JSONPathFieldSpec fromString(String name) { return JSONPathFieldSpec.createRootField(name); }
public static JSONPathFieldSpec createRootField(String name) { return new JSONPathFieldSpec(JSONPathFieldType.ROOT, name, null); }
public static JSONPathFieldSpec createJqField(String name, String expr) { return new JSONPathFieldSpec(JSONPathFieldType.JQ, name, expr); }
@JsonCreator public JSONFlatDataParser( @JacksonInject @Json ObjectMapper jsonMapper, @JsonProperty("keyFieldName") final String keyFieldName, @JsonProperty("valueFieldName") final String valueFieldName ) { Preconditions.checkArgument(!Strings.isNullOrEmpty(keyFieldName), "[keyFieldName] cannot be empty"); Preconditions.checkArgument(!Strings.isNullOrEmpty(valueFieldName), "[valueFieldName] cannot be empty"); this.keyFieldName = keyFieldName; this.valueFieldName = valueFieldName; // Copy jsonMapper; don't want to share canonicalization tables, etc., with the global ObjectMapper. this.parser = new DelegateParser( new JSONPathParser( new JSONPathSpec( false, ImmutableList.of( new JSONPathFieldSpec(JSONPathFieldType.ROOT, keyFieldName, keyFieldName), new JSONPathFieldSpec(JSONPathFieldType.ROOT, valueFieldName, valueFieldName) ) ), jsonMapper.copy() ), keyFieldName, valueFieldName ); }
true, ImmutableList.of( new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz", "baz"), new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz2", "baz2"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg", "$.o.mg"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg2", "$.o.mg2"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg", ".o.mg"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg2", ".o.mg2")
@Test public void testParseRowWithConditional() { final JSONParseSpec parseSpec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo")), null, null), new JSONPathSpec( true, ImmutableList.of( new JSONPathFieldSpec(JSONPathFieldType.PATH, "foo", "$.[?(@.maybe_object)].maybe_object.foo.test"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "baz", "$.maybe_object_2.foo.test"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "bar", "$.[?(@.something_else)].something_else.foo") ) ), null ); final Map<String, Object> expected = new HashMap<>(); expected.put("foo", new ArrayList()); expected.put("baz", null); expected.put("bar", Collections.singletonList("test")); final Parser<String, Object> parser = parseSpec.makeParser(); final Map<String, Object> parsedRow = parser.parseToMap("{\"something_else\": {\"foo\": \"test\"}}"); Assert.assertNotNull(parsedRow); Assert.assertEquals(expected, parsedRow); }
@Test public void testRejectDuplicates2() { List<JSONPathFieldSpec> fields = new ArrayList<>(); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "met-array", ".met.a")); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot have duplicate field definition: met-array"); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); jsonParser.parseToMap(nestedJson); }
@Test public void testRejectDuplicates() { List<JSONPathFieldSpec> fields = new ArrayList<>(); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot have duplicate field definition: met-array"); final Parser<String, Object> jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); jsonParser.parseToMap(nestedJson); }
fields.add(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "simpleVal", "simpleVal")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "timestamp", "timestamp")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested-foo.bar2", "$.foo.bar2")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "heybarx0", "$.hey[0].barx")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-heybarx0", ".hey[0].barx")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-met-array", ".met.a"));
fields.add(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "baz", "baz")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested-foo.bar1", "$.foo.bar1")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested-foo.bar2", "$.foo.bar2")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "heybarx0", "$.hey[0].barx")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "met-array", "$.met.a")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "testListConvert2", "testListConvert2")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "testMapConvert", "testMapConvert")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "INVALID_ROOT", "INVALID_ROOT_EXPR")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.PATH, "INVALID_PATH", "INVALID_PATH_EXPR")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-nested-foo.bar1", ".foo.bar1")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-heybarx0", ".hey[0].barx")); fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-met-array", ".met.a"));
public static JSONPathFieldSpec createNestedField(String name, String expr) { return new JSONPathFieldSpec(JSONPathFieldType.PATH, name, expr); }