try (final InputStream is = new FileInputStream(csvFile)) { try (final InputStreamReader reader = new InputStreamReader(is, charset)) { final Iterable<CSVRecord> records = csvFormat.withFirstRecordAsHeader().parse(reader); for (final CSVRecord record : records) { final String key = record.get(lookupKeyColumn); final String value = record.get(lookupValueColumn); if (StringUtils.isBlank(key)) { throw new IllegalStateException("Empty lookup key encountered in: " + csvFile);
.withHeader() .withDelimiter(delimiter) .withQuote(quote) .withRecordSeparator(recordSeparator); try (final CSVParser parser = new CSVParser(reader, format)) { ThreadMethods.throttledExecution(StreamMethods.enumerate(StreamMethods.stream(parser.spliterator(), false)).skip(skip).limit(limit), e -> { Integer rId = e.getKey(); CSVRecord row = e.getValue(); if (!row.isConsistent()) { logger.warn("WARNING: Skipping row {} because its size does not match the header size.", row.getRecordNumber()); TypeInference.DataType dataType = entry.getValue(); Object value = TypeInference.DataType.parse(row.get(column), dataType); //parse the string value according to the DataType if (yVariable != null && yVariable.equals(column)) { y = value;
CsvLineParser(char fieldDelimiter, char quote, char escape) { this.csvFormat = CSVFormat.DEFAULT .withIgnoreEmptyLines(true) .withDelimiter(fieldDelimiter) .withEscape(escape) .withQuote(quote); }
private static CSVFormat formatForDelimiter(char delimiter) { CSVFormat format = CSV_FORMAT; if (delimiter != format.getDelimiter()) { format = format.withDelimiter(delimiter); } return format; }
public static String[] parseAsCsv(String key, String value, Function<String, String> valueProcessor) { String cleanValue = MultivalueProperty.trimFieldsAndRemoveEmptyFields(value); List<String> result = new ArrayList<>(); try (CSVParser csvParser = CSVFormat.RFC4180 .withHeader((String) null) .withIgnoreEmptyLines() .withIgnoreSurroundingSpaces() .parse(new StringReader(cleanValue))) { List<CSVRecord> records = csvParser.getRecords(); if (records.isEmpty()) { return ArrayUtils.EMPTY_STRING_ARRAY; } processRecords(result, records, valueProcessor); return result.toArray(new String[result.size()]); } catch (IOException e) { throw new IllegalStateException("Property: '" + key + "' doesn't contain a valid CSV value: '" + value + "'", e); } }
public CSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader, final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding) throws IOException { super(logger, schema, hasHeader, ignoreHeader, dateFormat, timeFormat, timestampFormat); final Reader reader = new InputStreamReader(new BOMInputStream(in), encoding); CSVFormat withHeader; if (hasHeader) { withHeader = csvFormat.withSkipHeaderRecord(); if (ignoreHeader) { withHeader = withHeader.withHeader(schema.getFieldNames().toArray(new String[0])); } else { withHeader = withHeader.withFirstRecordAsHeader(); } } else { withHeader = csvFormat.withHeader(schema.getFieldNames().toArray(new String[0])); } csvParser = new CSVParser(reader, withHeader); }
format = format.withAllowMissingColumnNames(false); format = format.withCommentMarker('#'); format = format.withDelimiter(','); format = format.withEscape('\\'); format = format.withHeader("author", "title", "publishDate"); format = format.withHeaderComments("headerComment"); format = format.withNullString("NULL"); format = format.withIgnoreEmptyLines(true); format = format.withIgnoreSurroundingSpaces(true); format = format.withQuote('"'); format = format.withQuoteMode(QuoteMode.ALL); format = format.withRecordSeparator('\n'); format = format.withSkipHeaderRecord(false); try (final CSVParser parser = format.parse(getTestInput())) { for (final CSVRecord csvRecord : parser) { records++; if (csvRecord.hasComment()) { comments++;
@Override public Map<String, String> next() throws IOException { if (csvParser == null) { csvParser = CSVFormat.DEFAULT.withHeader().withDelimiter(delimiter).parse(inputStream); iterator = csvParser.iterator(); } if (!iterator.hasNext()) { return null; } CSVRecord csvRecord = iterator.next(); return csvRecord.toMap(); }
(double) totalHunspellComputationTime.get() / numMatches.get()))); SuggestionsOrdererConfig.setNgramsPath(args[1]); try (CSVParser parser = new CSVParser(new FileReader(args[0]), CSVFormat.DEFAULT.withFirstRecordAsHeader())) { for (CSVRecord record : parser) { String lang = record.get("language"); String covered = record.get("covered"); String replacement = record.get("replacement"); String sentenceStr = record.get("sentence");
public List<Tuple> read(InputStream is) throws IOException { CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(delim.charAt(0)); CSVParser csvParser = new CSVParser(new InputStreamReader(is), format); List<CSVRecord> records = csvParser.getRecords(); header = csvParser.getHeaderMap().entrySet().stream() .sorted((e1, e2) -> e1.getValue().compareTo(e2.getValue())) .map(Map.Entry::getKey).toArray(String[]::new); labelCol = labelCol == -1 ? records.get(0).size() - 1 : labelCol; List<Tuple> ds = records.stream().parallel().map(this::extractValuedFeat).collect(Collectors.toList()); return ds; }
assertTrue(testName + " require 1 param", split.length >= 1); CSVFormat format = CSVFormat.newFormat(',').withQuote('"'); boolean checkComments = false; for (int i = 1; i < split.length; i++) { final String[] option_parts = option.split("=", 2); if ("IgnoreEmpty".equalsIgnoreCase(option_parts[0])) { format = format.withIgnoreEmptyLines(Boolean.parseBoolean(option_parts[1])); } else if ("IgnoreSpaces".equalsIgnoreCase(option_parts[0])) { format = format.withIgnoreSurroundingSpaces(Boolean.parseBoolean(option_parts[1])); } else if ("CommentStart".equalsIgnoreCase(option_parts[0])) { format = format.withCommentMarker(option_parts[1].charAt(0)); } else if ("CheckComments".equalsIgnoreCase(option_parts[0])) { checkComments = true; try (final CSVParser parser = CSVParser.parse(new File(BASE, split[0]), Charset.defaultCharset(), format)) { for (final CSVRecord record : parser) { String parsed = Arrays.toString(record.values()); if (checkComments) { final String comment = record.getComment().replace("\n", "\\n"); if (comment != null) { parsed += "#" + comment; final int count = record.size(); assertEquals(testName, readTestData(), count + ":" + parsed);
private static CSVFormat buildCustomFormat(final PropertyContext context) { final char valueSeparator = getUnescapedChar(context, VALUE_SEPARATOR); CSVFormat format = CSVFormat.newFormat(valueSeparator) .withAllowMissingColumnNames() .withIgnoreEmptyLines(); format = format.withFirstRecordAsHeader(); format = format.withQuote(getChar(context, QUOTE_CHAR)); format = format.withEscape(getChar(context, ESCAPE_CHAR)); format = format.withTrim(context.getProperty(TRIM_FIELDS).asBoolean()); format = format.withCommentMarker(getChar(context, COMMENT_MARKER)); format = format.withNullString(CSVUtils.unescape(context.getProperty(NULL_STRING).getValue())); if (quoteValue != null) { final QuoteMode quoteMode = QuoteMode.valueOf(quoteValue.getValue()); format = format.withQuoteMode(quoteMode); if (trailingDelimiterValue != null) { final boolean trailingDelimiter = trailingDelimiterValue.asBoolean(); format = format.withTrailingDelimiter(trailingDelimiter); if (recordSeparator != null) { final String separator = unescape(recordSeparator.getValue()); format = format.withRecordSeparator(separator);
try (CSVParser parser = new CSVParser(new StringReader(content), getCSVFormat().withHeader())) { Set<String> columns = parser.getHeaderMap().keySet(); Map<String, Field> fieldMap = getFieldMap(objectType); for (String column : columns) { Field field = fieldMap.get(caseSensitiveFieldNames ? column : column.toLowerCase()); String value = record.get(column); Object object = objectFromString(value, field.getType()); field.set(o, object);
final CSVParser parser = new CSVParser(in, CSVFormat.DEFAULT.withHeader()); String group = record.get("QUERY_GROUP"); String query = record.get("QUERY"); String explain = record.get("EXPLAIN_PLAN"); String tenantId = record.get("TENANT_ID"); long avgTime = Long.parseLong(record.get("AVG_TIME_MS")); parser.close();
try (final InputStream is = new FileInputStream(csvFile)) { try (final InputStreamReader reader = new InputStreamReader(is, charset)) { final CSVParser records = csvFormat.withFirstRecordAsHeader().parse(reader); RecordSchema lookupRecordSchema = null; for (final CSVRecord record : records) { final String key = record.get(lookupKeyColumn); record.toMap().forEach((k, v) -> { if (!lookupKeyColumn.equals(k)) { properties.put(k, v);
private CSVParser createCSVParser(CSVParserSettings parserSettings) { CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(parserSettings.getDelimiter()) .withRecordSeparator(parserSettings.getRecordSeparator()) .withTrim(parserSettings.isTrim()) .withQuote(parserSettings.getQuote()) .withCommentMarker(parserSettings.getCommentMarker()) .withIgnoreEmptyLines(parserSettings.isIgnoreEmptyLines()) .withNullString(parserSettings.getNullString()); try { File file = new File(parserSettings.getPath()); return new CSVParser(new FileReader(file), csvFormat); } catch (Exception e) { throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); } } }
CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader(); if (theQuoteMode != null) { format = format.withQuote('"').withQuoteMode(theQuoteMode); parsed = new CSVParser(reader, format); Iterator<CSVRecord> iter = parsed.iterator(); ourLog.debug("Header map: {}", parsed.getHeaderMap()); while (iter.hasNext()) { CSVRecord nextRecord = iter.next(); if (nextRecord.isConsistent()==false) { continue;
try ( Reader reader = Files.newBufferedReader(Paths.get(file)); CSVParser csvParser = new CSVParser( reader, CSVFormat .DEFAULT .withRecordSeparator("\n") .withHeader(Header.class) .withFirstRecordAsHeader() .withIgnoreHeaderCase() .withIgnoreEmptyLines() .withTrim()); ) { retVal.setUrl(conceptMapUrl);
@Test public void testParseCustomNullValues() throws IOException { final StringWriter sw = new StringWriter(); final CSVFormat format = CSVFormat.DEFAULT.withNullString("NULL"); try (final CSVPrinter printer = new CSVPrinter(sw, format)) { printer.printRecord("a", null, "b"); } final String csvString = sw.toString(); assertEquals("a,NULL,b" + recordSeparator, csvString); try (final CSVParser iterable = format.parse(new StringReader(csvString))) { final Iterator<CSVRecord> iterator = iterable.iterator(); final CSVRecord record = iterator.next(); assertEquals("a", record.get(0)); assertEquals(null, record.get(1)); assertEquals("b", record.get(2)); assertFalse(iterator.hasNext()); } }
public GrblLookups(String prefix) { String filename = prefix + "_" + Localization.loadedLocale() + ".csv"; URL u = GrblLookups.class.getResource(pathFor(filename)); if (u == null) { filename = prefix + "_en_US.csv"; } try { try (BufferedReader reader = new BufferedReader( new InputStreamReader( GrblLookups.class.getResourceAsStream(pathFor(filename))))) { Iterable<CSVRecord> records = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(reader); for (CSVRecord record : records) { List<String> list = Lists.newArrayList(record.iterator()); lookups.put(record.get(0), list.toArray(new String[0])); } } } catch (IOException ex) { System.out.println("Unable to load GRBL resources."); ex.printStackTrace(); } }