@Override public boolean nextKeyValue() throws IOException, InterruptedException { boolean next = lineRecordReader.nextKeyValue(); if (next) { String line = lineRecordReader.getCurrentValue().toString(); try { JSONObject json = (JSONObject) parser.parse(line); String author = (String) json.get("author"); String link = (String) json.get("link"); } catch (ParseException e) { if (skipBadLines) { System.err.println("Bad line at offset: " + lineRecordReader.getCurrentKey().get() + ":\n" + e.getMessage()); badLineCount++; } else { throw new IOException(e); } } } return next; }
@Override public boolean nextKeyValue() throws IOException { if (!this.lineRecordReader.nextKeyValue()) return false; try (InputStream in = new ByteArrayInputStream(this.lineRecordReader.getCurrentValue().getBytes())) { this.vertexWritable.set(this.hasEdges ? this.graphsonReader.readVertex(in, Attachable::get, Attachable::get, Direction.BOTH) : this.graphsonReader.readVertex(in, Attachable::get)); return true; } }
@Override public boolean nextKeyValue() throws IOException { while (true) { if (!this.lineRecordReader.nextKeyValue()) return false; try { final Bindings bindings = this.engine.createBindings(); final StarGraph graph = StarGraph.open(); bindings.put(GRAPH, graph); bindings.put(LINE, this.lineRecordReader.getCurrentValue().toString()); final StarGraph.StarVertex sv = (StarGraph.StarVertex) script.eval(bindings); if (sv != null) { final Optional<StarGraph.StarVertex> vertex = sv.applyGraphFilter(this.graphFilter); if (vertex.isPresent()) { this.vertexWritable.set(vertex.get()); return true; } } } catch (final ScriptException e) { throw new IOException(e.getMessage()); } } }
if (!this.lrr.nextKeyValue()) { return false;
public ArrayList<String> readRecords(URL testFileUrl, int splitSize) throws IOException { // Set up context File testFile = new File(testFileUrl.getFile()); long testFileSize = testFile.length(); Path testFilePath = new Path(testFile.getAbsolutePath()); Configuration conf = new Configuration(); conf.setInt("io.file.buffer.size", 1); TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); // Gather the records returned by the record reader ArrayList<String> records = new ArrayList<String>(); long offset = 0; while (offset < testFileSize) { FileSplit split = new FileSplit(testFilePath, offset, splitSize, null); LineRecordReader reader = new LineRecordReader(); reader.initialize(split, context); while (reader.nextKeyValue()) { records.add(reader.getCurrentValue().toString()); } offset += splitSize; } return records; }
boolean firstLine = true; boolean skipBOM = true; while (reader.nextKeyValue()) { if (firstLine) { firstLine = false;
@Override public boolean nextKeyValue() throws IOException, InterruptedException { return lineRecordReader.nextKeyValue(); } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { return lineRecordReader.nextKeyValue(); } }
@Test public void testMultipleClose() throws IOException { URL testFileUrl = getClass().getClassLoader(). getResource("recordSpanningMultipleSplits.txt.bz2"); assertNotNull("Cannot find recordSpanningMultipleSplits.txt.bz2", testFileUrl); File testFile = new File(testFileUrl.getFile()); Path testFilePath = new Path(testFile.getAbsolutePath()); long testFileSize = testFile.length(); Configuration conf = new Configuration(); conf.setInt(org.apache.hadoop.mapreduce.lib.input. LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE); TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); // read the data and check whether BOM is skipped FileSplit split = new FileSplit(testFilePath, 0, testFileSize, null); LineRecordReader reader = new LineRecordReader(); reader.initialize(split, context); //noinspection StatementWithEmptyBody while (reader.nextKeyValue()) ; reader.close(); reader.close(); BZip2Codec codec = new BZip2Codec(); codec.setConf(conf); Set<Decompressor> decompressors = new HashSet<Decompressor>(); for (int i = 0; i < 10; ++i) { decompressors.add(CodecPool.getDecompressor(codec)); } assertEquals(10, decompressors.size()); }
LongWritable key; Text value; reader.nextKeyValue(); key = reader.getCurrentKey(); value = reader.getCurrentValue(); reader.nextKeyValue(); assertFalse(reader.nextKeyValue()); reader.initialize(split, context); reader.nextKeyValue(); key = reader.getCurrentKey(); value = reader.getCurrentValue(); assertFalse(reader.nextKeyValue()); reader = new LineRecordReader(null); reader.initialize(split, context); reader.nextKeyValue(); key = reader.getCurrentKey(); value = reader.getCurrentValue(); reader.nextKeyValue(); assertFalse(reader.nextKeyValue());
reader.initialize(split, context); int numRecordsNoSplits = 0; while (reader.nextKeyValue()) { ++numRecordsNoSplits; reader.initialize(split, context); int numRecordsFirstSplit = 0; while (reader.nextKeyValue()) { ++numRecordsFirstSplit; reader.initialize(split, context); int numRecordsRemainingSplits = 0; while (reader.nextKeyValue()) { ++numRecordsRemainingSplits;
assertTrue("Expected record got nothing", reader.nextKeyValue()); LongWritable key = reader.getCurrentKey(); Text value = reader.getCurrentValue(); assertTrue("Expected record got nothing", reader.nextKeyValue()); assertEquals("Wrong length for record value", 2, value.getLength()); assertTrue("Expected record got nothing", reader.nextKeyValue()); assertEquals("Wrong length for record value", 3, value.getLength()); assertFalse(reader.nextKeyValue()); assertFalse("Unexpected record returned", reader.nextKeyValue()); key = reader.getCurrentKey(); assertNull("Unexpected key returned", key); assertTrue("Expected record got nothing", reader.nextKeyValue()); key = reader.getCurrentKey(); value = reader.getCurrentValue(); assertFalse(reader.nextKeyValue()); assertEquals("Wrong position after record read", 11, key.get()); reader = new LineRecordReader(recordDelimiterBytes); reader.initialize(split, context); assertTrue("Expected record got nothing", reader.nextKeyValue()); key = reader.getCurrentKey(); value = reader.getCurrentValue();
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (reader.nextKeyValue()) { //<co id="ch02_comment_csv_inputformat7"/> loadCSV(); //<co id="ch02_comment_csv_inputformat8"/> return true; } else { value = null; return false; } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (reader.nextKeyValue()) { parseLine(); return true; } else { value = null; return false; } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (reader.nextKeyValue()) { parseLine(); return true; } else { value = null; return false; } }
@Override public boolean nextKeyValue() throws IOException { if (reader.nextKeyValue()) { outValue.set(reader.getCurrentValue()); outKey.set(reader.getCurrentKey().get()); return true; } else { return false; } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { while (reader.nextKeyValue()) { if ((value_ = logReader.decodeLine(reader.getCurrentValue())) != null) { return true; } } return false; }
@Override public boolean nextKeyValue() throws IOException { if (!this.lineRecordReader.nextKeyValue()) return false; try (InputStream in = new ByteArrayInputStream(this.lineRecordReader.getCurrentValue().getBytes())) { this.vertexWritable.set(this.hasEdges ? this.graphsonReader.readVertex(in, Attachable::get, Attachable::get, Direction.BOTH) : this.graphsonReader.readVertex(in, Attachable::get)); return true; } }
@Override public boolean nextKeyValue() throws IOException { if (!this.lineRecordReader.nextKeyValue()) return false; this.vertex = FaunusGraphSONUtility.fromJSON(this.lineRecordReader.getCurrentValue().toString()); this.vertexQuery.defaultFilter(this.vertex); this.vertex.enablePath(this.pathEnabled); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { while (reader.nextKeyValue()) { value_.clear(); if (decodeLineToJson(jsonParser_, reader.getCurrentValue(), value_)) { return true; } } return false; }