@Deprecated public ColumnConfig(String name, Type type, String format) { this.name = name; this.type = type; this.option = Exec.newConfigSource(); // only for backward compatibility if (format != null) { option.set("format", format); } }
@Before public void setup() throws Exception { config = Exec.newConfigSource(); }
@Before public void setup() { config = Exec.newConfigSource() .set("newline", "LF") .set("columns", ImmutableList.of( ImmutableMap.<String,Object>of( "name", "date_code", "type", "string", "option", ImmutableMap.of()), ImmutableMap.<String,Object>of( "name", "foo", "type", "string", "option", ImmutableMap.of())) ); reloadPluginTask(); }
@Test public void checkFirstCharacterTypesRuleNeitherReplacePrefix() { HashMap<String, Object> parameters = new HashMap<>(); parameters.put("rule", "first_character_types"); ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.copyOf(parameters))); exception.expect(ConfigException.class); // TODO(dmikurube): Except "Caused by": exception.expectCause(instanceOf(JsonMappingException.class)); // Needs to import org.hamcrest.Matchers... in addition to org.junit... renameAndCheckSchema(config, new String[0], new String[0]); }
@Test public void checkTruncateRuleDefault() { final String[] original = { "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890" }; final String[] expected = { "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678" }; ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "truncate"))); renameAndCheckSchema(config, original, expected); }
@Test public void checkRuleUpperToLowerRule() { final String[] original = { "_C0", "_C1", "_c2" }; final String[] expected = { "_c0", "_c1", "_c2" }; ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "upper_to_lower"))); renameAndCheckSchema(config, original, expected); }
@Test(expected = ConfigException.class) public void checkColumnsRequired() { ConfigSource config = Exec.newConfigSource(); config.loadConfig(CsvParserPlugin.PluginTask.class); }
@Test public void checkTruncateRule() { final String[] original = { "foo", "bar", "gj", "foobar", "foobarbaz" }; final String[] expected = { "foo", "bar", "gj", "foo", "foo" }; ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "truncate", "max_length", "3"))); renameAndCheckSchema(config, original, expected); }
@Test public void checkRuleLowerToUpperRule() { final String[] original = { "_C0", "_C1", "_c2" }; final String[] expected = { "_C0", "_C1", "_C2" }; ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "lower_to_upper"))); renameAndCheckSchema(config, original, expected); }
@Test public void checkTruncateRuleNegative() { final String[] original = { "foo" }; ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "truncate", "max_length", -1))); exception.expect(TaskValidationException.class); // TODO(dmikurube): Except "Caused by": exception.expectCause(instanceOf(JsonMappingException.class)); // Needs to import org.hamcrest.Matchers... in addition to org.junit... renameAndCheckSchema(config, original, original); }
@Test public void checkDefaultValues() { PluginTask task = Exec.newConfigSource().loadConfig(PluginTask.class); assertTrue(task.getRenameMap().isEmpty()); }
private static LineDecoder.DecoderTask getExampleConfig(Charset charset, Newline newline, LineDelimiter lineDelimiter) { ConfigSource config = Exec.newConfigSource() .set("charset", charset) .set("newline", newline); if (lineDelimiter != null) { config.set("line_delimiter_recognized", lineDelimiter); } return config.loadConfig(LineDecoder.DecoderTask.class); }
private LineEncoder newEncoder(String charset, String newline, FileOutput output) throws Exception { ConfigSource config = Exec.newConfigSource() .set("charset", charset) .set("newline", newline); return new LineEncoder(output, config.loadConfig(LineEncoder.EncoderTask.class)); }
@Test public void testSimpleFormat() throws Exception { ConfigSource config = Exec.newConfigSource() .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S.%9N %z"); // %Z is OS-dependent FormatterTestTask task = config.loadConfig(FormatterTestTask.class); TimestampFormatter formatter = TimestampFormatter.of(task, Optional.<TimestampFormatter.TimestampColumnOption>absent()); assertEquals("2014-11-19 02:46:29.123456000 +0000", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456 * 1000))); }
@Test public void testDefaultValues() { ConfigSource config = Exec.newConfigSource(); LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class); assertEquals(StandardCharsets.UTF_8, task.getCharset()); assertEquals(Newline.CRLF, task.getNewline()); }
@Test public void checkDefaultValues() { ConfigSource config = Exec.newConfigSource(); JsonParserPlugin.PluginTask task = config.loadConfig(JsonParserPlugin.PluginTask.class); assertEquals(false, task.getStopOnInvalidRecord()); assertEquals(JsonParserPlugin.InvalidEscapeStringPolicy.PASSTHROUGH, task.getInvalidEscapeStringPolicy()); }
@Test public void testSimpleParse() throws Exception { ConfigSource config = Exec.newConfigSource() .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S %z"); // %Z is OS-dependent ParserTestTask task = config.loadConfig(ParserTestTask.class); TimestampParser parser = TimestampParserLegacy.createTimestampParserForTesting(task); assertEquals(Timestamp.ofEpochSecond(1416365189, 0), parser.parse("2014-11-19 02:46:29 +0000")); }
@Test public void testLoadConfig() { ConfigSource config = Exec.newConfigSource() .set("charset", "utf-16") .set("newline", "CRLF") .set("line_delimiter_recognized", "LF"); LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class); assertEquals(StandardCharsets.UTF_16, task.getCharset()); assertEquals(Newline.CRLF, task.getNewline()); assertEquals(LineDelimiter.LF, task.getLineDelimiterRecognized().get()); }
@Test public void testDefaultDate() throws Exception { ConfigSource config = Exec.newConfigSource() .set("default_timestamp_format", "%H:%M:%S %Z") .set("default_date", "2016-02-03"); ParserTestTask ptask = config.loadConfig(ParserTestTask.class); TimestampParser parser = TimestampParserLegacy.createTimestampParserForTesting(ptask); assertEquals(Timestamp.ofEpochSecond(1454467589, 0), parser.parse("02:46:29 +0000")); } }
@Test public void testUnixtimeFormat() throws Exception { ConfigSource config = Exec.newConfigSource() .set("default_timestamp_format", "%s"); FormatterTestTask ftask = config.loadConfig(FormatterTestTask.class); TimestampFormatter formatter = TimestampFormatter.of(ftask, Optional.<TimestampFormatter.TimestampColumnOption>absent()); assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189))); ParserTestTask ptask = config.loadConfig(ParserTestTask.class); TimestampParser parser = TimestampParserLegacy.createTimestampParserForTesting(ptask); assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189")); }