@Test public void testFieldDelimiterTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setFieldDelimiter("@"); String tableOptions = fileFormat.toTableOptions(); assertContains("fieldDelimiter => '@'", tableOptions); }
final TextFileConfig textFileConfig = (TextFileConfig)this; stringBuilder.append("type => 'text', "); stringBuilder.append(format("fieldDelimiter => %s, ", SqlUtils.stringLiteral(textFileConfig.getFieldDelimiter()))); stringBuilder.append(format("comment => %s, ", SqlUtils.stringLiteral(singleChar(textFileConfig.getComment())))); stringBuilder.append(format("%1$sescape%1$s => %2$s, ", SqlUtils.QUOTE, SqlUtils.stringLiteral(singleChar(textFileConfig.getEscape())))); stringBuilder.append(format("quote => %s, ", SqlUtils.stringLiteral(singleChar(textFileConfig.getQuote())))); stringBuilder.append(format("lineDelimiter => %s, ", SqlUtils.stringLiteral(textFileConfig.getLineDelimiter()))); stringBuilder.append(format("extractHeader => %s, ", textFileConfig.getExtractHeader().toString())); stringBuilder.append(format("skipFirstLine => %s, ", textFileConfig.getSkipFirstLine().toString())); stringBuilder.append(format("autoGenerateColumnNames => %s, ", textFileConfig.getAutoGenerateColumnNames().toString())); stringBuilder.append(format("trimHeader => %s", textFileConfig.getTrimHeader().toString())); return stringBuilder.toString();
@Test public void testCommentTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setComment("$"); String tableOptions = fileFormat.toTableOptions(); assertContains("comment => '$'", tableOptions); }
@Test public void testExtractHeaderTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setExtractHeader(true); String tableOptions = fileFormat.toTableOptions(); assertContains("extractHeader => true", tableOptions); }
@Test public void testEscapeTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setEscape("\\"); String tableOptions = fileFormat.toTableOptions(); assertContains("\"escape\" => '\\", tableOptions); }
@Test public void testPreviewTinyAcqWithHeader() throws Exception { TextFileConfig fileConfig = new TextFileConfig(); fileConfig.setFieldDelimiter("|"); fileConfig.setLineDelimiter("\n"); fileConfig.setName("tinyacq.txt"); fileConfig.setExtractHeader(true); String fileUrlPath = getUrlPath("/datasets/tinyacq.txt"); doc("preview data for source file"); JobDataFragment data = expectSuccess(getBuilder(getAPIv2().path("/source/dacfs_test/file_preview/" + fileUrlPath)).buildPost(Entity.json(fileConfig)), JobDataFragment.class); assertEquals(23, data.getColumns().size()); assertEquals(getCurrentDremioDaemon().getBindingProvider().lookup(SabotContext.class).getOptionManager().getOption(FormatTools.TARGET_RECORDS), data.getReturnedRowCount()); }
final TextFileConfig textFileConfig = new TextFileConfig(); TextParsingSettings settings = new TextParsingSettings(); settings.set((TextFormatConfig) formatPlugin.getConfig()); textFileConfig.setComment(new Character((char) settings.getComment()).toString()); textFileConfig.setEscape(new Character((char) settings.getQuoteEscape()).toString()); textFileConfig.setFieldDelimiter(new Character((char) settings.getDelimiter()).toString()); textFileConfig.setQuote(new Character((char) settings.getQuote()).toString()); textFileConfig.setExtractHeader(settings.isHeaderExtractionEnabled()); textFileConfig.setSkipFirstLine(settings.isSkipFirstLine()); textFileConfig.setLineDelimiter(new String(settings.getNewLineDelimiter())); textFileConfig.setAutoGenerateColumnNames(settings.isAutoGenerateColumnNames()); textFileConfig.setTrimHeader(settings.isTrimHeader()); return textFileConfig;
namespaceService.addOrUpdateSource(new SourcePath("src").toNamespaceKey(), source.asSourceConfig()); TextFileConfig fileConfig = new TextFileConfig(); fileConfig.setComment("#"); fileConfig.setFieldDelimiter("|"); fileConfig.setExtractHeader(true); fileConfig.setName("fff"); "/source/src/file_format/file1")).buildGet(), FileFormatUI.class).getFileFormat(); assertEquals(fileConfig.getName(), format1.getName()); assertEquals(fileConfig.getFieldDelimiter(), format1.getFieldDelimiter()); assertEquals(fileConfig.getExtractHeader(), format1.getExtractHeader()); assertEquals(fileConfig.asFileConfig().getType(), format1.asFileConfig().getType()); assertEquals(fileConfig.asFileConfig().getOwner(), format1.asFileConfig().getOwner()); expectSuccess(getBuilder(getAPIv2().path("/source/src/file_format/file1").queryParam("version", format1.getVersion())).buildDelete());
assertEquals(folderFormatUrl, defaultFormat.getLinks().get("self")); TextFileConfig fileConfig = new TextFileConfig(); fileConfig.setComment("#"); fileConfig.setFieldDelimiter("|"); fileConfig.setName("fff"); fileConfig.setVersion(null); FileFormat fileFormat1 = FileFormat.getForFolder(fileConfig.asFileConfig()); fileFormat1.setName("tmp._dac.folder1"); FileFormat fileFormat2 = FileFormat.getForFolder(fileConfig.asFileConfig()); fileFormat2.setName("tmp._dac.folder2"); fileConfig.setFieldDelimiter(","); fileConfig.setVersion(fileFormat1.getVersion()); fileFormat1 = FileFormat.getForFolder(fileConfig.asFileConfig()); assertEquals(",", ((TextFileConfig)fileFormat1).getFieldDelimiter());
@Test public void testLineDelimiterTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setLineDelimiter("\t"); String tableOptions = fileFormat.toTableOptions(); assertContains("lineDelimiter => '\t'", tableOptions); }
private static void runTests(HomeFileConf homeFileStore) throws Exception { // text file Path textFile = new Path(FileUtils.getResourceAsFile("/datasets/text/comma.txt").getAbsolutePath()); uploadFile(homeFileStore, textFile, "comma", "txt", new TextFileConfig().setFieldDelimiter(","), null); Path csvFile = new Path(FileUtils.getResourceAsFile("/datasets/csv/comma.csv").getAbsolutePath()); uploadFile(homeFileStore, csvFile, "comma1", "csv", new TextFileConfig().setFieldDelimiter(","), null); Path jsonFile = new Path(FileUtils.getResourceAsFile("/datasets/users.json").getAbsolutePath()); uploadFile(homeFileStore, jsonFile, "users", "json", new JsonFileConfig(), null); Path excelFile = new Path(FileUtils.getResourceAsFile("/testfiles/excel.xlsx").getAbsolutePath()); uploadFile(homeFileStore, excelFile, "excel", "xlsx", new ExcelFileConfig(), null); // query files runQuery("comma", 4, 3, null); runQuery("comma1", 4, 3, null); runQuery("users", 3, 2, null); runQuery("excel", 6, 5, null); // add file to folder FolderPath folderPath = new FolderPath(ImmutableList.of(HomeName.getUserHomePath(DEFAULT_USER_NAME).getName(), "testupload")); newNamespaceService().addOrUpdateFolder(folderPath.toNamespaceKey(), new FolderConfig() .setName("testupload") .setFullPathList(folderPath.toPathList())); uploadFile(homeFileStore, textFile, "comma", "txt", new TextFileConfig().setFieldDelimiter(","), folderPath); runQuery("comma", 4, 3, folderPath); }
@Test public void testSingleQuoteTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setQuote("'"); String tableOptions = fileFormat.toTableOptions(); assertContains("quote => ''''", tableOptions); }
TextFileConfig textFileConfig = new TextFileConfig(); textFileConfig.setLineDelimiter("\n"); dataset = createPDS(folder.getPath(), textFileConfig);
@Test public void testsetSkipFirstLineTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setSkipFirstLine(true); String tableOptions = fileFormat.toTableOptions(); assertContains("skipFirstLine => true", tableOptions); }
@Test public void testAutoGenerateColumnNamesTextFile() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); fileFormat.setAutoGenerateColumnNames(false); String tableOptions = fileFormat.toTableOptions(); assertContains("autoGenerateColumnNames => false", tableOptions); }
TextFileConfig fileConfig = new TextFileConfig(); fileConfig.setFieldDelimiter(","); fileConfig.setName("comma.txt");
@Test // DX-5410 public void formatChangeForUploadedHomeFile() throws Exception { FormDataMultiPart form = new FormDataMultiPart(); FormDataBodyPart fileBody = new FormDataBodyPart("file", FileUtils.getResourceAsFile("/datasets/csv/pipe.csv"), MediaType.MULTIPART_FORM_DATA_TYPE); form.bodyPart(fileBody); form.bodyPart(new FormDataBodyPart("fileName", "pipe")); doc("uploading a text file"); File file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_start/").queryParam("extension", "csv")) .buildPost(Entity.entity(form, form.getMediaType())), File.class); file1 = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/upload_finish/pipe")) .buildPost(Entity.json(file1.getFileFormat().getFileFormat())), File.class); final FileFormat defaultFileFormat = file1.getFileFormat().getFileFormat(); assertTrue(defaultFileFormat instanceof TextFileConfig); assertEquals(",", ((TextFileConfig)defaultFileFormat).getFieldDelimiter()); doc("change the format settings of uploaded file"); final TextFileConfig newFileFormat = (TextFileConfig)defaultFileFormat; newFileFormat.setFieldDelimiter("|"); final FileFormat updatedFileFormat = expectSuccess(getBuilder(getAPIv2().path("home/" + HOME_NAME + "/file_format/pipe")) .buildPut(Entity.json(newFileFormat)), FileFormatUI.class).getFileFormat(); assertTrue(updatedFileFormat instanceof TextFileConfig); assertEquals("|", ((TextFileConfig)updatedFileFormat).getFieldDelimiter()); }
public TextFileConfig newMessage() { return new TextFileConfig(); }
@Test public void testDefaultTextFileFormatOptions() throws Exception { TextFileConfig fileFormat = new TextFileConfig(); String tableOptions = fileFormat.toTableOptions(); assertContains("type => 'text'", tableOptions); assertContains("fieldDelimiter => ','", tableOptions); assertContains("comment => '#'", tableOptions); assertContains("\"escape\" => '\"'", tableOptions); assertContains("quote => '\"'", tableOptions); assertContains("lineDelimiter => '\r\n'", tableOptions); assertContains("extractHeader => false", tableOptions); assertContains("skipFirstLine => false", tableOptions); assertContains("autoGenerateColumnNames => true", tableOptions); }
@Test public void testCommaSeparatedTextFile() throws Exception { TextFileConfig fileConfig = new TextFileConfig(); fileConfig.setFieldDelimiter(","); fileConfig.setLineDelimiter("\n"); fileConfig.setName("comma.txt"); String fileUrlPath = getUrlPath("/datasets/text/comma.txt"); String fileParentUrlPath = getUrlPath("/datasets/text/"); doc("preview data for source file"); JobDataFragment data = expectSuccess(getBuilder(getAPIv2().path("/source/dacfs_test/file_preview" + fileUrlPath)).buildPost(Entity.json(fileConfig)), JobDataFragment.class); assertEquals(4, data.getReturnedRowCount()); assertEquals(3, data.getColumns().size()); checkCounts(fileParentUrlPath, "comma.txt", false /* false because we have not saved dataset yet */, 0, 0, 0); // previews are internal queries }