public JsonFileConfig newMessage() { return new JsonFileConfig(); }
@Test public void testDefaultJsonFileFormatOptions() throws Exception { JsonFileConfig fileFormat = new JsonFileConfig(); String tableOptions = fileFormat.toTableOptions(); assertContains("type => 'json'", tableOptions); }
dacSample1.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample1.setName(allTypesJson); dacSample1.setPhysicalDataset(new PhysicalDataset().setFormatSettings(new JsonFileConfig().asFileConfig())); namespaceService.addOrUpdateDataset(dacSampleAllTypes.toNamespaceKey(), dacSample1); dacSample1.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample1.setName("dac-sample1.json"); dacSample1.setPhysicalDataset(new PhysicalDataset().setFormatSettings(new JsonFileConfig().asFileConfig())); namespaceService.addOrUpdateDataset(dacSample1Path.toNamespaceKey(), dacSample1); dacSample2.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample2.setName("dac-sample2.json"); dacSample2.setPhysicalDataset(new PhysicalDataset().setFormatSettings(new JsonFileConfig().asFileConfig())); namespaceService.addOrUpdateDataset(dacSample2Path.toNamespaceKey(), dacSample2);
null, null, new JsonFileConfig(), null ); config.setCreatedAt(dataset.getCreatedAt()); PhysicalDataset physicalDataset = new PhysicalDataset(); physicalDataset.setFormatSettings(new JsonFileConfig().asFileConfig()); config.setType(PHYSICAL_DATASET); config.setPhysicalDataset(physicalDataset);
public static FileFormat getEmptyConfig(FileType type) { switch (type) { case TEXT: case CSV: case TSV: case PSV: return new TextFileConfig(); case JSON: return new JsonFileConfig(); case PARQUET: return new ParquetFileConfig(); case AVRO: return new AvroFileConfig(); case HTTP_LOG: return new HttpLogFileConfig(); case EXCEL: return new ExcelFileConfig(); case XLS: return new XlsFileConfig(); case UNKNOWN: break; default: break; } return null; }
return new JsonFileConfig();
@Test public void testSchemaChangeExit() throws Exception { String fileUrlPath = getUrlPath("/datasets/schemachange"); JsonFileConfig jsonFileConfig = new JsonFileConfig(); doc("preview json source file"); JobDataFragment data = expectSuccess(getBuilder(getAPIv2().path("/source/dacfs_test/folder_preview" + fileUrlPath)).buildPost(Entity.json(jsonFileConfig)), JobDataFragment.class); assertEquals(1, data.getReturnedRowCount()); assertEquals(1, data.getColumns().size()); }
private static void runTests(HomeFileConf homeFileStore) throws Exception { // text file Path textFile = new Path(FileUtils.getResourceAsFile("/datasets/text/comma.txt").getAbsolutePath()); uploadFile(homeFileStore, textFile, "comma", "txt", new TextFileConfig().setFieldDelimiter(","), null); Path csvFile = new Path(FileUtils.getResourceAsFile("/datasets/csv/comma.csv").getAbsolutePath()); uploadFile(homeFileStore, csvFile, "comma1", "csv", new TextFileConfig().setFieldDelimiter(","), null); Path jsonFile = new Path(FileUtils.getResourceAsFile("/datasets/users.json").getAbsolutePath()); uploadFile(homeFileStore, jsonFile, "users", "json", new JsonFileConfig(), null); Path excelFile = new Path(FileUtils.getResourceAsFile("/testfiles/excel.xlsx").getAbsolutePath()); uploadFile(homeFileStore, excelFile, "excel", "xlsx", new ExcelFileConfig(), null); // query files runQuery("comma", 4, 3, null); runQuery("comma1", 4, 3, null); runQuery("users", 3, 2, null); runQuery("excel", 6, 5, null); // add file to folder FolderPath folderPath = new FolderPath(ImmutableList.of(HomeName.getUserHomePath(DEFAULT_USER_NAME).getName(), "testupload")); newNamespaceService().addOrUpdateFolder(folderPath.toNamespaceKey(), new FolderConfig() .setName("testupload") .setFullPathList(folderPath.toPathList())); uploadFile(homeFileStore, textFile, "comma", "txt", new TextFileConfig().setFieldDelimiter(","), folderPath); runQuery("comma", 4, 3, folderPath); }
Dataset dataset = createPDS(CatalogServiceHelper.getPathFromInternalId(file.getId()), new JsonFileConfig());
Dataset dataset = createPDS(CatalogServiceHelper.getPathFromInternalId(file.getId()), new JsonFileConfig());
Dataset dataset = createPDS(CatalogServiceHelper.getPathFromInternalId(file.getId()), new JsonFileConfig());
@Test public void testJsonFile() throws Exception { final JobsService jobsService = l(JobsService.class); JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(createQuery("/datasets/users.json")) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE)); JobDataFragment jobData = job.getData().truncate(500); assertEquals(3, jobData.getReturnedRowCount()); assertEquals(2, jobData.getColumns().size()); String fileUrlPath = getUrlPath("/datasets/users.json"); String fileParentUrlPath = getUrlPath("/datasets/"); JsonFileConfig jsonFileConfig = new JsonFileConfig(); doc("preview json source file"); JobDataFragment data = expectSuccess(getBuilder(getAPIv2().path("/source/dacfs_test/file_preview" + fileUrlPath)).buildPost(Entity.json(jsonFileConfig)), JobDataFragment.class); assertEquals(3, data.getReturnedRowCount()); assertEquals(2, data.getColumns().size()); doc("creating dataset from source file"); InitialPreviewResponse createResponse = expectSuccess(getBuilder(getAPIv2().path( "source/dacfs_test/new_untitled_from_file/" + getUrlPath("/datasets/users.json"))).buildPost(Entity.json("")), InitialPreviewResponse.class); assertEquals(2, createResponse.getData().getColumns().size()); checkCounts(fileParentUrlPath, "users.json", true, 2, 0, 0); }