public LineEncoder(FileOutput out, EncoderTask task) { this.newline = task.getNewline().getString(); this.underlyingFileOutput = out; this.outputStream = new FileOutputOutputStream(underlyingFileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH_FINISH); CharsetEncoder encoder = task.getCharset() .newEncoder() .onMalformedInput(CodingErrorAction.REPLACE) // TODO configurable? .onUnmappableCharacter(CodingErrorAction.REPLACE); // TODO configurable? this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoder), 32 * 1024); }
private static FileInput newFileInputFromLines(CsvParserPlugin.PluginTask task, String... lines) { List<Buffer> buffers = new ArrayList<>(); for (String line : lines) { byte[] buffer = (line + task.getNewline().getString()).getBytes(task.getCharset()); buffers.add(Buffer.wrap(buffer)); } return new ListFileInput(ImmutableList.of(buffers)); }
@Test public void testEscapeQuote() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class, CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class); method.setAccessible(true); CsvFormatterPlugin formatter = new CsvFormatterPlugin(); char delimiter = ','; CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL; String newline = Newline.LF.getString(); assertEquals("\"AB\\\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '\\', newline, "")); assertEquals("\"AB\"\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '"', newline, "")); }
public CsvTokenizer(LineDecoder input, CsvParserPlugin.PluginTask task) { String delimiter = task.getDelimiter(); if (delimiter.length() == 0) { throw new ConfigException("Empty delimiter is not allowed"); } else { this.delimiterChar = delimiter.charAt(0); if (delimiter.length() > 1) { delimiterFollowingString = delimiter.substring(1); } else { delimiterFollowingString = null; } } quote = task.getQuoteChar().or(CsvParserPlugin.QuoteCharacter.noQuote()).getCharacter(); escape = task.getEscapeChar().or(CsvParserPlugin.EscapeCharacter.noEscape()).getCharacter(); newline = task.getNewline().getString(); trimIfNotQuoted = task.getTrimIfNotQuoted(); quotesInQuotedFields = task.getQuotesInQuotedFields(); if (trimIfNotQuoted && quotesInQuotedFields != QuotesInQuotedFields.ACCEPT_ONLY_RFC4180_ESCAPED) { // The combination makes some syntax very ambiguous such as: // val1, \"\"val2\"\" ,val3 throw new ConfigException("[quotes_in_quoted_fields != ACCEPT_ONLY_RFC4180_ESCAPED] is not allowed to specify with [trim_if_not_quoted = true]"); } maxQuotedSizeLimit = task.getMaxQuotedSizeLimit(); commentLineMarker = task.getCommentLineMarker().orNull(); nullStringOrNull = task.getNullString().orNull(); this.input = input; }
char escape = '"'; CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL; String newline = Newline.LF.getString(); String nullString = "";
char escape = '\\'; CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.NONE; String newline = Newline.LF.getString(); String nullString = "";
char escape = '"'; CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.ALL; String newline = Newline.LF.getString(); String nullString = "";
newline = Newline.LF.getString(); testCases = ImmutableList.of( ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\r\nDEF"), newline = Newline.CRLF.getString(); testCases = ImmutableList.of( ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\r\nDEF"), newline = Newline.CR.getString(); testCases = ImmutableList.of( ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\r\nDEF"),
@Test public void testNullString() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class, CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class); method.setAccessible(true); CsvFormatterPlugin formatter = new CsvFormatterPlugin(); char delimiter = ','; char quote = '"'; char escape = '"'; CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL; String newline = Newline.LF.getString(); assertEquals("\"\"", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "")); assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "")); assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "")); assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "")); assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A")); assertEquals("\"N/A\"", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A")); assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A")); assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A")); } }
final char quote = task.getQuoteChar() != '\0' ? task.getQuoteChar() : '"'; final char escape = task.getEscapeChar().or(quotePolicy == QuotePolicy.NONE ? '\\' : quote); final String newlineInField = task.getNewlineInField().getString(); final String nullString = task.getNullString();