public static Set<HiveRecordCursorProvider> getDefaultHiveRecordCursorProvider(HiveClientConfig hiveClientConfig) { HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return ImmutableSet.<HiveRecordCursorProvider>builder() .add(new GenericHiveRecordCursorProvider(testHdfsEnvironment)) .build(); }
@Test public void testIsS3FileSystem() { HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); assertTrue(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("s3://test-bucket/test-folder"))); assertFalse(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("/test-dir/test-folder"))); }
@Test public void testIsViewFileSystem() { HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); Path viewfsPath = new Path("viewfs://ns-default/test-folder"); Path nonViewfsPath = new Path("hdfs://localhost/test-dir/test-folder"); // ViewFS check requires the mount point config hdfsEnvironment.getConfiguration(CONTEXT, viewfsPath).set("fs.viewfs.mounttable.ns-default.link./test-folder", "hdfs://localhost/app"); assertTrue(isViewFileSystem(CONTEXT, hdfsEnvironment, viewfsPath)); assertFalse(isViewFileSystem(CONTEXT, hdfsEnvironment, nonViewfsPath)); } }
public static Set<HivePageSourceFactory> getDefaultHiveDataStreamFactories(HiveClientConfig hiveClientConfig) { FileFormatDataSourceStats stats = new FileFormatDataSourceStats(); HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return ImmutableSet.<HivePageSourceFactory>builder() .add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) .add(new OrcPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats)) .add(new DwrfPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) .add(new ParquetPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) .build(); }
public static Set<HiveFileWriterFactory> getDefaultHiveFileWriterFactories(HiveClientConfig hiveClientConfig) { HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return ImmutableSet.<HiveFileWriterFactory>builder() .add(new RcFileFileWriterFactory(testHdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), hiveClientConfig, new FileFormatDataSourceStats())) .add(getDefaultOrcFileWriterFactory(hiveClientConfig)) .build(); }
private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, File outputFile) { Properties splitProperties = new Properties(); splitProperties.setProperty(FILE_INPUT_FORMAT, config.getHiveStorageFormat().getInputFormat()); splitProperties.setProperty(SERIALIZATION_LIB, config.getHiveStorageFormat().getSerDe()); splitProperties.setProperty("columns", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getName).collect(toList()))); splitProperties.setProperty("columns.types", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getHiveType).map(hiveType -> hiveType.getHiveTypeName().toString()).collect(toList()))); HiveSplit split = new HiveSplit( SCHEMA_NAME, TABLE_NAME, "", "file:///" + outputFile.getAbsolutePath(), 0, outputFile.length(), outputFile.length(), splitProperties, ImmutableList.of(), ImmutableList.of(), OptionalInt.empty(), false, TupleDomain.all(), ImmutableMap.of(), Optional.empty(), false); HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER); return provider.createPageSource(transaction, getSession(config), split, ImmutableList.copyOf(getColumnHandles())); }
public static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HiveClientConfig hiveClientConfig) { HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return new OrcFileWriterFactory( testHdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), hiveClientConfig, new FileFormatDataSourceStats(), new OrcFileWriterConfig()); }
ImmutableMap.of()); JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class); HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(config); HivePageSinkProvider provider = new HivePageSinkProvider( getDefaultHiveFileWriterFactories(config),
public static Set<HiveRecordCursorProvider> getDefaultHiveRecordCursorProvider(HiveClientConfig hiveClientConfig) { HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return ImmutableSet.<HiveRecordCursorProvider>builder() .add(new GenericHiveRecordCursorProvider(testHdfsEnvironment)) .build(); }
@Test public void testIsS3FileSystem() { HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); assertTrue(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("s3://test-bucket/test-folder"))); assertFalse(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("/test-dir/test-folder"))); }
@Test public void testIsViewFileSystem() { HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); Path viewfsPath = new Path("viewfs://ns-default/test-folder"); Path nonViewfsPath = new Path("hdfs://localhost/test-dir/test-folder"); // ViewFS check requires the mount point config hdfsEnvironment.getConfiguration(CONTEXT, viewfsPath).set("fs.viewfs.mounttable.ns-default.link./test-folder", "hdfs://localhost/app"); assertTrue(isViewFileSystem(CONTEXT, hdfsEnvironment, viewfsPath)); assertFalse(isViewFileSystem(CONTEXT, hdfsEnvironment, nonViewfsPath)); } }
public static Set<HivePageSourceFactory> getDefaultHiveDataStreamFactories(HiveClientConfig hiveClientConfig) { FileFormatDataSourceStats stats = new FileFormatDataSourceStats(); HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return ImmutableSet.<HivePageSourceFactory>builder() .add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) .add(new OrcPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats)) .add(new DwrfPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) .add(new ParquetPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) .build(); }
public static Set<HiveFileWriterFactory> getDefaultHiveFileWriterFactories(HiveClientConfig hiveClientConfig) { HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return ImmutableSet.<HiveFileWriterFactory>builder() .add(new RcFileFileWriterFactory(testHdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), hiveClientConfig, new FileFormatDataSourceStats())) .add(getDefaultOrcFileWriterFactory(hiveClientConfig)) .build(); }
private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, File outputFile) { Properties splitProperties = new Properties(); splitProperties.setProperty(FILE_INPUT_FORMAT, config.getHiveStorageFormat().getInputFormat()); splitProperties.setProperty(SERIALIZATION_LIB, config.getHiveStorageFormat().getSerDe()); splitProperties.setProperty("columns", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getName).collect(toList()))); splitProperties.setProperty("columns.types", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getHiveType).map(hiveType -> hiveType.getHiveTypeName().toString()).collect(toList()))); HiveSplit split = new HiveSplit( SCHEMA_NAME, TABLE_NAME, "", "file:///" + outputFile.getAbsolutePath(), 0, outputFile.length(), outputFile.length(), splitProperties, ImmutableList.of(), ImmutableList.of(), OptionalInt.empty(), false, TupleDomain.all(), ImmutableMap.of(), Optional.empty(), false); HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER); return provider.createPageSource(transaction, getSession(config), split, ImmutableList.copyOf(getColumnHandles())); }
public static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HiveClientConfig hiveClientConfig) { HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); return new OrcFileWriterFactory( testHdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), hiveClientConfig, new FileFormatDataSourceStats(), new OrcFileWriterConfig()); }
ImmutableMap.of()); JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class); HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(config); HivePageSinkProvider provider = new HivePageSinkProvider( getDefaultHiveFileWriterFactories(config),