@Test public void testScrollMultiValueList() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("list")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(1, read.size()); Object[] objects = read.get(0); Map map = (Map) read.get(0)[1]; List links = (List) map.get("links"); assertTrue(links.contains(null)); }
protected Object read(String fieldName, Token t, String fieldMapping, Parser parser) { if (t == Token.START_ARRAY) { return list(fieldName, fieldMapping, parser); FieldType esType = mapping(fieldMapping, parser); if ((esType != null && esType.equals(FieldType.NESTED)) || isArrayField(fieldMapping)) { return singletonList(fieldMapping, map(fieldMapping, parser), parser); } else { return map(fieldMapping, parser); FieldType esType = mapping(fieldMapping, parser); if (isArrayField(fieldMapping)) { return singletonList(fieldMapping, parseValue(parser, esType), parser); } else { return parseValue(parser, esType);
protected Object readListItem(String fieldName, Token t, String fieldMapping, Parser parser) { if (t == Token.START_ARRAY) { return list(fieldName, fieldMapping, parser); } // handle nested nodes first else if (t == Token.START_OBJECT) { // Don't need special handling for nested fields since this field is already in an array. return map(fieldMapping, parser); } FieldType esType = mapping(fieldMapping, parser); if (t.isValue()) { String rawValue = parser.text(); try { return parseValue(parser, esType); } catch (Exception ex) { throw new EsHadoopParsingException(String.format(Locale.ROOT, "Cannot parse value [%s] for field [%s]", rawValue, fieldName), ex); } } return null; }
protected Object list(String fieldName, String fieldMapping, Parser parser) { Token t = parser.currentToken(); if (t == null) { t = parser.nextToken(); } if (t == Token.START_ARRAY) { t = parser.nextToken(); } Object array = reader.createArray(mapping(fieldMapping, parser)); // create only one element since with fields, we always get arrays which create unneeded allocations List<Object> content = new ArrayList<Object>(1); for (; parser.currentToken() != Token.END_ARRAY;) { content.add(readListItem(fieldName, parser.currentToken(), fieldMapping, parser)); } // eliminate END_ARRAY parser.nextToken(); array = reader.addToArray(array, content); return array; }
Scroll scroll(String scrollId, ScrollReader reader) throws IOException { InputStream scroll = client.scroll(scrollId); try { return reader.read(scroll); } finally { if (scroll instanceof StatsAware) { stats.aggregate(((StatsAware) scroll).stats()); } } }
if (FieldType.isGeo(mapping(fieldMapping, parser))) { toggleGeo = true; insideGeo = true; if (shouldSkip(absoluteName)) { Token nt = parser.nextToken(); if (nt.isValue()) { reader.addToMap(map, fieldName, read(absoluteName, parser.nextToken(), nodeMapping, parser)); reader.endField(absoluteName);
public ScrollReaderJsonTest(boolean readMetadata, String metadataField, boolean readAsJson) { this.readMetadata = readMetadata; this.readAsJson = readAsJson; this.metadataField = metadataField; this.mapper = new ObjectMapper(); reader = new ScrollReader(getScrollCfg()); }
readResult = readHitAsJson(workingParser); } else { readResult = readHitAsMap(workingParser);
if (FieldType.isGeo(mapping(fieldMapping))) { toggleGeo = true; insideGeo = true; if (shouldSkip(absoluteName)) { skipCurrentBlock(); reader.addToMap(map, fieldName, read(absoluteName, parser.nextToken(), nodeMapping)); reader.endField(absoluteName);
protected Object list(String fieldName, String fieldMapping) { Token t = parser.currentToken(); if (t == null) { t = parser.nextToken(); } if (t == Token.START_ARRAY) { t = parser.nextToken(); } Object array = reader.createArray(mapping(fieldMapping)); // create only one element since with fields, we always get arrays which create unneeded allocations List<Object> content = new ArrayList<Object>(1); for (; parser.currentToken() != Token.END_ARRAY;) { content.add(read(fieldName, parser.currentToken(), fieldMapping)); } // eliminate END_ARRAY parser.nextToken(); array = reader.addToArray(array, content); return array; }
protected Object singletonList(String fieldMapping, Object value, Parser parser) { Object array = reader.createArray(mapping(fieldMapping, parser)); // create only one element since with fields, we always get arrays which create unneeded allocations List<Object> content = new ArrayList<Object>(1); content.add(value); array = reader.addToArray(array, content); return array; }
protected Object read(String fieldName, Token t, String fieldMapping) { if (t == Token.START_ARRAY) { return list(fieldName, fieldMapping); } // handle nested nodes first else if (t == Token.START_OBJECT) { return map(fieldMapping); } FieldType esType = mapping(fieldMapping); if (t.isValue()) { String rawValue = parser.text(); try { return parseValue(esType); } catch (Exception ex) { throw new EsHadoopParsingException(String.format(Locale.ROOT, "Cannot parse value [%s] for field [%s]", rawValue, fieldName), ex); } } return null; }
public Scroll read(InputStream content) throws IOException { Assert.notNull(content); //copy content BytesArray copy = IOUtils.asBytes(content); content = new FastByteArrayInputStream(copy); if (log.isTraceEnabled()) { log.trace("About to parse scroll content " + copy); } Parser parser = new JacksonJsonParser(content); try { return read(parser, copy); } finally { parser.close(); } }
if (FieldType.isGeo(mapping(fieldMapping, parser))) { toggleGeo = true; insideGeo = true; if (shouldSkip(absoluteName)) { Token nt = parser.nextToken(); if (nt.isValue()) { reader.addToMap(map, fieldName, read(absoluteName, parser.nextToken(), nodeMapping, parser)); reader.endField(absoluteName);
public ScrollReaderTest(boolean readMetadata, String metadataField) { this.readMetadata = readMetadata; this.metadataField = metadataField; reader = new ScrollReader(getScrollReaderCfg()); }
private Object[] readHit() { Token t = parser.currentToken(); Assert.isTrue(t == Token.START_OBJECT, "expected object, found " + t); return (returnRawJson ? readHitAsJson() : readHitAsMap()); }
protected Object list(String fieldName, String fieldMapping, Parser parser) { Token t = parser.currentToken(); if (t == null) { t = parser.nextToken(); } if (t == Token.START_ARRAY) { t = parser.nextToken(); } Object array = reader.createArray(mapping(fieldMapping, parser)); // create only one element since with fields, we always get arrays which create unneeded allocations List<Object> content = new ArrayList<Object>(1); for (; parser.currentToken() != Token.END_ARRAY;) { content.add(readListItem(fieldName, parser.currentToken(), fieldMapping, parser)); } // eliminate END_ARRAY parser.nextToken(); array = reader.addToArray(array, content); return array; }
protected Object singletonList(String fieldMapping, Object value, Parser parser) { Object array = reader.createArray(mapping(fieldMapping, parser)); // create only one element since with fields, we always get arrays which create unneeded allocations List<Object> content = new ArrayList<Object>(1); content.add(value); array = reader.addToArray(array, content); return array; }