runner.transaction(config, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
ConfigDiff guessed; try { input.transaction(guessInputConfig, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) { if (taskCount == 0) {
runner.transaction(samplingInputConfig, new InputPlugin.Control() { public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount) { if (taskCount == 0) {
runner.transaction(config, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
private void doTest(ConfigSource config) { ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); assertRecords(config, output); }
@Test public void usePathAsHighPriorityThanPathPrefix() { ConfigSource config = this.config.deepCopy() .set("path", String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX)) .set("path_prefix", "foo"); // path_prefix has the bad value, if path_prefix is chosen, expected result will be failed ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX), configDiff.get(String.class, "last_path")); assertRecords(config, output); }
@Test public void usePath() { ConfigSource config = this.config.deepCopy() .set("path", String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX)) .set("path_prefix", null); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX), configDiff.get(String.class, "last_path")); assertRecords(config, output); }
@Test public void useTotalFileCountLimit() throws Exception { ConfigSource config = this.config.deepCopy().set("total_file_count_limit", 0); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertNull(configDiff.get(String.class, "last_path")); assertEquals(0, getRecords(config, output).size()); }
@Test public void useLastPath() throws Exception { ConfigSource config = this.config.deepCopy().set("last_path", EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv"); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); assertEquals(0, getRecords(config, output).size()); }
@Test public void simpleTest() { ConfigSource config = this.config.deepCopy(); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); assertRecords(config, output); }
@Test public void useIncremental() { ConfigSource config = this.config.deepCopy().set("incremental", false); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertFalse(configDiff.has("last_path")); }
@Test public void emptyFilesWithLastPath() throws Exception { ConfigSource config = this.config.deepCopy() .set("path_prefix", "empty_files_prefix") .set("last_path", EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv"); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); // keep the last_path assertEquals(0, getRecords(config, output).size()); }
@Test public void usePathMatchPattern() throws Exception { { // match pattern ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/sample_01"); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); assertRecords(config, output); } output = new MockPageOutput(); { // not match pattern ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/match/"); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertNull(configDiff.get(String.class, "last_path")); assertEquals(0, getRecords(config, output).size()); } }