ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of())))); final MockPageOutput output = new MockPageOutput();
ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of())))); final MockPageOutput output = new MockPageOutput(); runner.transaction(config, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource,
public static List<Page> buildPage(BufferAllocator bufferAllocator, Schema schema, Object... values) { MockPageOutput output = new MockPageOutput(); try (PageBuilder builder = new PageBuilder(bufferAllocator, schema, output)) {
private List<Object[]> filter(PluginTask task, Schema inputSchema, Object ... objects) { MockPageOutput output = new MockPageOutput(); Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema); PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output); PageReader pageReader = new PageReader(inputSchema); ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder); List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects); for (Page page : pages) { pageReader.setPage(page); while (pageReader.nextRecord()) { outputSchema.visitColumns(visitor); pageBuilder.addRecord(); } } pageBuilder.finish(); pageBuilder.close(); return Pages.toObjects(outputSchema, output.pages); }
@Test public void usePathMatchPattern() throws Exception { { // match pattern ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/sample_01"); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); assertRecords(config, output); } output = new MockPageOutput(); { // not match pattern ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/match/"); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertNull(configDiff.get(String.class, "last_path")); assertEquals(0, getRecords(config, output).size()); } }
@Before public void createResources() { config = runtime.getExec().newConfigSource() .set("type", "s3") .set("bucket", EMBULK_S3_TEST_BUCKET) .set("access_key_id", EMBULK_S3_TEST_ACCESS_KEY_ID) .set("secret_access_key", EMBULK_S3_TEST_SECRET_ACCESS_KEY) .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX) .set("parser", parserConfig(schemaConfig())); runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class)); output = new MockPageOutput(); }
@Before public void createResources() { config = runtime.getExec().newConfigSource() .set("type", "s3") .set("bucket", EMBULK_S3_TEST_BUCKET) .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX) .set("parser", parserConfig(schemaConfig())); runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class)); output = new TestPageBuilderReader.MockPageOutput(); }
private List<Page> buildPages(Schema schema, final Object... objects) { MockPageOutput output = new MockPageOutput(); this.builder = new PageBuilder(bufferAllocator, schema, output); int idx = 0;
@Test public void testDoubleWriteStringsToRow() { MockPageOutput output = new MockPageOutput(); Schema schema = Schema.builder() .add("col0", STRING) .add("col1", STRING) .add("col2", STRING) .build(); builder = new PageBuilder(bufferAllocator, schema, output); builder.setString(0, "v0"); builder.setString(1, "v1"); builder.setNull(2); builder.setString(0, "v2"); // stored to page for col0 builder.setNull(1); // null is stored to page for col1 builder.setString(2, "v3"); // stored to page for col2 builder.addRecord(); builder.finish(); builder.close(); reader = new PageReader(schema); reader.setPage(output.pages.get(0)); assertTrue(reader.nextRecord()); assertEquals(reader.getString(0), "v2"); assertTrue(reader.isNull(1)); assertEquals(reader.getString(2), "v3"); assertFalse(reader.nextRecord()); reader.close(); }
@Test public void testDoubleWriteJsonsToRow() { MockPageOutput output = new MockPageOutput(); Schema schema = Schema.builder() .add("col0", JSON) .add("col1", JSON) .add("col2", JSON) .build(); builder = new PageBuilder(bufferAllocator, schema, output); builder.setJson(0, newString("v0")); builder.setJson(1, newString("v1")); builder.setNull(2); builder.setJson(0, newString("v2")); // store to page for col0 builder.setNull(1); // null is stored to page for col1 builder.setJson(2, newString("v3")); // store to page for col2 builder.addRecord(); builder.finish(); builder.close(); reader = new PageReader(schema); reader.setPage(output.pages.get(0)); assertTrue(reader.nextRecord()); assertEquals(reader.getJson(0), newString("v2")); assertTrue(reader.isNull(1)); assertEquals(reader.getJson(2), newString("v3")); assertFalse(reader.nextRecord()); reader.close(); }
@Test public void testEmptySchema() { MockPageOutput output = new MockPageOutput(); this.builder = new PageBuilder(bufferAllocator, Schema.builder().build(), output); builder.addRecord(); builder.addRecord(); builder.flush(); builder.close(); this.reader = new PageReader(Schema.builder().build()); assertEquals(1, output.pages.size()); reader.setPage(output.pages.get(0)); assertTrue(reader.nextRecord()); assertTrue(reader.nextRecord()); assertFalse(reader.nextRecord()); }
@Before public void createResource() { config = config(); plugin = new JsonParserPlugin(); output = new MockPageOutput(); }
@Test public void testRepeatableFlush() { MockPageOutput output = new MockPageOutput(); this.builder = new PageBuilder(bufferAllocator, Schema.builder().add("col1", STRING).build(), output); builder.flush(); builder.flush(); } }
@Test public void testRepeatableClose() { MockPageOutput output = new MockPageOutput(); this.builder = new PageBuilder(bufferAllocator, Schema.builder().add("col1", STRING).build(), output); builder.close(); builder.close(); }