@Override public boolean hasNext() { if (self.nextLine != null) { return true; } else { self.nextLine = self.poll(); return self.nextLine != null; } }
public boolean nextFile() { boolean next = input.nextFile(); if (next) { lineNumber = 0; } return next; }
public boolean nextFile() { boolean has = inputStream.nextFile(); if (has && charset.equals(UTF_8)) { skipBom(); } return has; }
private static LineDecoder newDecoder(Charset charset, Newline newline, LineDelimiter lineDelimiter, List<Buffer> buffers) { ListFileInput input = new ListFileInput(ImmutableList.of(buffers)); return new LineDecoder(input, getExampleConfig(charset, newline, lineDelimiter)); }
private static List<String> doDecode(Charset charset, Newline newline, LineDelimiter lineDelimiter, List<Buffer> buffers) { ImmutableList.Builder<String> builder = ImmutableList.builder(); LineDecoder decoder = newDecoder(charset, newline, lineDelimiter, buffers); decoder.nextFile(); while (true) { String line = decoder.poll(); if (line == null) { break; } builder.add(line); } return builder.build(); }
private static List<List<String>> parse(CsvParserPlugin.PluginTask task, FileInput input) { LineDecoder decoder = new LineDecoder(input, task); CsvTokenizer tokenizer = new CsvTokenizer(decoder, task); Schema schema = task.getSchemaConfig().toSchema(); tokenizer.nextFile(); List<List<String>> records = new ArrayList<>(); while (tokenizer.nextRecord()) { List<String> record = new ArrayList<>(); for (Column c : schema.getColumns()) { String v = tokenizer.nextColumnOrNull(); record.add(v); } records.add(record); } return records; }
final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig()); final JsonParser jsonParser = new JsonParser(); final CsvTokenizer tokenizer = new CsvTokenizer(new LineDecoder(input, task), task); final boolean allowOptionalColumns = task.getAllowOptionalColumns(); final boolean allowExtraColumns = task.getAllowExtraColumns();
public boolean skipHeaderLine() { boolean skipped = input.poll() != null; if (skipped) { lineNumber++; } return skipped; }
}; FileInput input = newFileInputFromLines(task, lines); LineDecoder decoder = new LineDecoder(input, task); CsvTokenizer tokenizer = new CsvTokenizer(decoder, task); Schema schema = task.getSchemaConfig().toSchema();
private boolean nextLine(boolean skipEmptyLine) { while (true) { if (!unreadLines.isEmpty()) { line = unreadLines.removeFirst(); } else { line = input.poll(); if (line == null) { return false; } } linePos = 0; lineNumber++; boolean skip = skipEmptyLine && (line.isEmpty() || (commentLineMarker != null && line.startsWith(commentLineMarker))); if (!skip) { return true; } } }