@Inject public HdfsConfigurationUpdater(HiveClientConfig config, S3ConfigurationUpdater s3ConfigurationUpdater) { requireNonNull(config, "config is null"); checkArgument(config.getDfsTimeout().toMillis() >= 1, "dfsTimeout must be at least 1 ms"); checkArgument(toIntExact(config.getTextMaxLineLength().toBytes()) >= 1, "textMaxLineLength must be at least 1 byte"); this.socksProxy = config.getMetastoreSocksProxy(); this.ipcPingInterval = config.getIpcPingInterval(); this.dfsTimeout = config.getDfsTimeout(); this.dfsConnectTimeout = config.getDfsConnectTimeout(); this.dfsConnectMaxRetries = config.getDfsConnectMaxRetries(); this.domainSocketPath = config.getDomainSocketPath(); this.resourcesConfiguration = readConfiguration(config.getResourceConfigFiles()); this.compressionCodec = config.getHiveCompressionCodec(); this.fileSystemMaxCacheSize = config.getFileSystemMaxCacheSize(); this.isHdfsWireEncryptionEnabled = config.isHdfsWireEncryptionEnabled(); this.textMaxLineLength = toIntExact(config.getTextMaxLineLength().toBytes()); this.s3ConfigurationUpdater = requireNonNull(s3ConfigurationUpdater, "s3ConfigurationUpdater is null"); }
private static String makeFileName(File tempDir, HiveClientConfig config) { return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getHiveCompressionCodec().name(); }
@Inject public HivePageSinkProvider( HdfsEnvironment hdfsEnvironment, HiveMetastore metastore, PageIndexerFactory pageIndexerFactory, TypeManager typeManager, HiveClientConfig config, LocationService locationService, JsonCodec<PartitionUpdate> partitionUpdateCodec) { this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.metastore = requireNonNull(metastore, "metastore is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.maxOpenPartitions = config.getMaxPartitionsPerWriter(); this.immutablePartitions = config.isImmutablePartitions(); this.compressed = config.getHiveCompressionCodec() != HiveCompressionCodec.NONE; this.locationService = requireNonNull(locationService, "locationService is null"); this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null"); }
private static String makeFileName(File tempDir, HiveClientConfig config) { return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getHiveCompressionCodec().name(); }
private static String makeFileName(File tempDir, HiveClientConfig config) { return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getHiveCompressionCodec().name(); }
@Inject public HdfsConfigurationUpdater(HiveClientConfig hiveClientConfig) { requireNonNull(hiveClientConfig, "hiveClientConfig is null"); checkArgument(hiveClientConfig.getDfsTimeout().toMillis() >= 1, "dfsTimeout must be at least 1 ms"); this.socksProxy = hiveClientConfig.getMetastoreSocksProxy(); this.dfsTimeout = hiveClientConfig.getDfsTimeout(); this.dfsConnectTimeout = hiveClientConfig.getDfsConnectTimeout(); this.dfsConnectMaxRetries = hiveClientConfig.getDfsConnectMaxRetries(); this.domainSocketPath = hiveClientConfig.getDomainSocketPath(); this.s3AwsAccessKey = hiveClientConfig.getS3AwsAccessKey(); this.s3AwsSecretKey = hiveClientConfig.getS3AwsSecretKey(); this.s3UseInstanceCredentials = hiveClientConfig.isS3UseInstanceCredentials(); this.s3SslEnabled = hiveClientConfig.isS3SslEnabled(); this.s3SseEnabled = hiveClientConfig.isS3SseEnabled(); this.s3MaxClientRetries = hiveClientConfig.getS3MaxClientRetries(); this.s3MaxErrorRetries = hiveClientConfig.getS3MaxErrorRetries(); this.s3MaxBackoffTime = hiveClientConfig.getS3MaxBackoffTime(); this.s3MaxRetryTime = hiveClientConfig.getS3MaxRetryTime(); this.s3ConnectTimeout = hiveClientConfig.getS3ConnectTimeout(); this.s3SocketTimeout = hiveClientConfig.getS3SocketTimeout(); this.s3MaxConnections = hiveClientConfig.getS3MaxConnections(); this.s3MultipartMinFileSize = hiveClientConfig.getS3MultipartMinFileSize(); this.s3MultipartMinPartSize = hiveClientConfig.getS3MultipartMinPartSize(); this.s3StagingDirectory = hiveClientConfig.getS3StagingDirectory(); this.resourcePaths = hiveClientConfig.getResourceConfigFiles(); this.pinS3ClientToCurrentRegion = hiveClientConfig.isPinS3ClientToCurrentRegion(); this.compressionCodec = hiveClientConfig.getHiveCompressionCodec(); }