public static FileHiveMetastore createTestingFileHiveMetastore(File catalogDirectory) { HiveClientConfig hiveClientConfig = new HiveClientConfig(); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); return new FileHiveMetastore(hdfsEnvironment, catalogDirectory.toURI().toString(), "test"); }
public void updateConfiguration(Configuration config) { copy(resourcesConfiguration, config); // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local config.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, NoOpDNSToSwitchMapping.class, DNSToSwitchMapping.class); if (socksProxy != null) { config.setClass(HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, SocksSocketFactory.class, SocketFactory.class); config.set(HADOOP_SOCKS_SERVER_KEY, socksProxy.toString()); } if (domainSocketPath != null) { config.setStrings(DFS_DOMAIN_SOCKET_PATH_KEY, domainSocketPath); } // only enable short circuit reads if domain socket path is properly configured if (!config.get(DFS_DOMAIN_SOCKET_PATH_KEY, "").trim().isEmpty()) { config.setBooleanIfUnset(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); } config.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, toIntExact(dfsTimeout.toMillis())); config.setInt(IPC_PING_INTERVAL_KEY, toIntExact(ipcPingInterval.toMillis())); config.setInt(IPC_CLIENT_CONNECT_TIMEOUT_KEY, toIntExact(dfsConnectTimeout.toMillis())); config.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, dfsConnectMaxRetries); if (isHdfsWireEncryptionEnabled) { config.set(HADOOP_RPC_PROTECTION, "privacy"); config.setBoolean("dfs.encrypt.data.transfer", true); } config.setInt("fs.cache.max-size", fileSystemMaxCacheSize); config.setInt(LineRecordReader.MAX_LINE_LENGTH, textMaxLineLength); configureCompression(config, compressionCodec); s3ConfigurationUpdater.updateConfiguration(config); }
@Override protected Configuration initialValue() { Configuration configuration = new Configuration(false); copy(INITIAL_CONFIGURATION, configuration); updater.updateConfiguration(configuration); return configuration; } };
@Inject public HdfsConfigurationUpdater(HiveClientConfig config, S3ConfigurationUpdater s3ConfigurationUpdater) { requireNonNull(config, "config is null"); checkArgument(config.getDfsTimeout().toMillis() >= 1, "dfsTimeout must be at least 1 ms"); checkArgument(toIntExact(config.getTextMaxLineLength().toBytes()) >= 1, "textMaxLineLength must be at least 1 byte"); this.socksProxy = config.getMetastoreSocksProxy(); this.ipcPingInterval = config.getIpcPingInterval(); this.dfsTimeout = config.getDfsTimeout(); this.dfsConnectTimeout = config.getDfsConnectTimeout(); this.dfsConnectMaxRetries = config.getDfsConnectMaxRetries(); this.domainSocketPath = config.getDomainSocketPath(); this.resourcesConfiguration = readConfiguration(config.getResourceConfigFiles()); this.compressionCodec = config.getHiveCompressionCodec(); this.fileSystemMaxCacheSize = config.getFileSystemMaxCacheSize(); this.isHdfsWireEncryptionEnabled = config.isHdfsWireEncryptionEnabled(); this.textMaxLineLength = toIntExact(config.getTextMaxLineLength().toBytes()); this.s3ConfigurationUpdater = requireNonNull(s3ConfigurationUpdater, "s3ConfigurationUpdater is null"); }
public TestingHdfsEnvironment() { super( new HiveHdfsConfiguration(new HdfsConfigurationUpdater(new HiveClientConfig())), new HiveClientConfig(), new NoHdfsAuthentication()); }
public RecordFormatWriter(File targetFile, List<String> columnNames, List<Type> columnTypes, HiveCompressionCodec compressionCodec, HiveStorageFormat format, ConnectorSession session) { JobConf config = new JobConf(conf); configureCompression(config, compressionCodec); recordWriter = new RecordFileWriter( new Path(targetFile.toURI()), columnNames, fromHiveStorageFormat(format), createSchema(format, columnNames, columnTypes), format.getEstimatedWriterSystemMemoryUsage(), config, TYPE_MANAGER, session); }
public static KerberosHadoopAuthentication createKerberosHadoopAuthentication(KerberosAuthentication kerberosAuthentication, HdfsConfigurationUpdater updater) { Configuration configuration = getInitialConfiguration(); updater.updateConfiguration(configuration); // In order to enable KERBEROS authentication method for HDFS // UserGroupInformation.authenticationMethod static field must be set to KERBEROS // It is further used in many places in DfsClient configuration.set("hadoop.security.authentication", "kerberos"); UserGroupInformation.setConfiguration(configuration); return new KerberosHadoopAuthentication(kerberosAuthentication); }
HdfsConfiguration createHdfsConfiguration(HiveClientConfig config) { S3ConfigurationUpdater s3Config = new PrestoS3ConfigurationUpdater(new HiveS3Config() .setS3AwsAccessKey(awsAccessKey) .setS3AwsSecretKey(awsSecretKey)); return new HiveHdfsConfiguration(new HdfsConfigurationUpdater(config, s3Config)); }
configureCompression(jobConf, compressionCodec);
@Override protected Configuration initialValue() { Configuration config = new Configuration(false); for (Map.Entry<String, String> entry : DEFAULT_CONFIGURATION) { config.set(entry.getKey(), entry.getValue()); } updater.updateConfiguration(config); return config; } };
@Override protected ExtendedHiveMetastore createMetastore(File tempDir) { File baseDir = new File(tempDir, "metastore"); HiveClientConfig hiveConfig = new HiveClientConfig(); HdfsConfigurationUpdater updator = new HdfsConfigurationUpdater(hiveConfig); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(updator); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveConfig, new NoHdfsAuthentication()); return new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test"); }
configureCompression(jobConf, compressionCodec);
/** * GlueHiveMetastore currently uses AWS Default Credential Provider Chain, * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default * on ways to set your AWS credentials which will be needed to run this test. */ @Override protected ExtendedHiveMetastore createMetastore(File tempDir) { HiveClientConfig hiveClientConfig = new HiveClientConfig(); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); GlueHiveMetastoreConfig glueConfig = new GlueHiveMetastoreConfig(); glueConfig.setDefaultWarehouseDir(tempDir.toURI().toString()); return new GlueHiveMetastore(hdfsEnvironment, glueConfig); }
configureCompression(config, compressionCodec);
public static HdfsEnvironment createTestHdfsEnvironment(HiveClientConfig config) { HdfsConfiguration hdfsConfig = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(config, new PrestoS3ConfigurationUpdater(new HiveS3Config()))); return new HdfsEnvironment(hdfsConfig, config, new NoHdfsAuthentication()); }
public RecordFormatWriter(File targetFile, List<String> columnNames, List<Type> columnTypes, HiveCompressionCodec compressionCodec, HiveStorageFormat format, ConnectorSession session) { JobConf config = new JobConf(conf); configureCompression(config, compressionCodec); recordWriter = new RecordFileWriter( new Path(targetFile.toURI()), columnNames, fromHiveStorageFormat(format), createSchema(format, columnNames, columnTypes), format.getEstimatedWriterSystemMemoryUsage(), config, TYPE_MANAGER, session); }
HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication());
public RecordFormatWriter(File targetFile, List<String> columnNames, List<Type> columnTypes, HiveCompressionCodec compressionCodec, HiveStorageFormat format) { JobConf config = new JobConf(conf); configureCompression(config, compressionCodec); List<DataColumn> dataColumns = new ArrayList<>(columnNames.size()); for (int i = 0; i < columnNames.size(); i++) { dataColumns.add(new DataColumn(columnNames.get(i), columnTypes.get(i), HiveType.toHiveType(columnTypes.get(i)))); } recordWriter = new HiveRecordWriter( "test_schema", "test_table", null, compressionCodec != HiveCompressionCodec.NONE, true, dataColumns, format.getOutputFormat(), format.getSerDe(), createSchema(format, columnNames, columnTypes), targetFile.getName(), targetFile.getParent(), targetFile.toString(), TYPE_MANAGER, config); }
@Test public void testFileSystemCache() throws IOException { ImpersonatingHdfsAuthentication auth = new ImpersonatingHdfsAuthentication(new SimpleHadoopAuthentication()); HdfsEnvironment environment = new HdfsEnvironment( new HiveHdfsConfiguration(new HdfsConfigurationUpdater(new HiveClientConfig())), new HiveClientConfig(), auth); FileSystem fs1 = getFileSystem(environment, "user"); FileSystem fs2 = getFileSystem(environment, "user"); assertSame(fs1, fs2); FileSystem fs3 = getFileSystem(environment, "other_user"); assertNotSame(fs1, fs3); FileSystem fs4 = getFileSystem(environment, "other_user"); assertSame(fs3, fs4); }
configureCompression(jobConf, compressionCodec);