public FileSourceConfiguration(@NonNull final Configuration conf) { ConfigUtil.checkMandatoryProperties(conf, getMandatoryProperties()); this.conf = conf; }
public HoodieConfiguration(@NonNull final Configuration conf, @NotEmpty final String tableKey, @NonNull final Optional<String> version) { this.conf = conf; this.tableKey = tableKey; this.version = version; ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); }
public HDFSSchemaServiceConfiguration(@NonNull final Configuration conf) { ConfigUtil.checkMandatoryProperties(conf, getMandatoryProperties()); this.conf = conf; }
public KafkaConfiguration(@NonNull final Configuration conf) { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); this.kafkaParams = new HashMap<>(); this.kafkaParams.put(GROUP_ID, DEFAULT_GROUP_ID); this.kafkaParams.putAll(getConf().getPropertiesWithPrefix(KAFKA_CONNECTION_PREFIX, true)); this.kafkaParams.put(KEY_DESERIALIZER, ByteArrayDeserializer.class.getCanonicalName()); this.kafkaParams.put(VALUE_DESERIALIZER, ByteArrayDeserializer.class.getCanonicalName()); this.kafkaParams.put(ENABLE_AUTO_COMMIT, ENABLE_AUTO_COMMIT_VALUE); // If retry backoff is not set then we would want to reduce it to lower values. Default value is 400ms. if (!kafkaParams.containsKey(KAFKA_BACKOFF_MS_CONFIG)) { kafkaParams.put(KAFKA_BACKOFF_MS_CONFIG, DEFAULT_KAFKA_BACKOFF_MS_CONFIG); } }
public ZookeeperConfiguration(@NonNull final Configuration conf) { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.getConf(), getMandatoryProperties()); this.zkQuorum = this.getConf().getProperty(ZK_QUORUM).get(); this.zkPort = this.getConf().getProperty(ZK_PORT).get(); }
public ErrorTableConfiguration(@NonNull final Configuration conf) { this.conf = conf; this.isEnabled = conf.getBooleanProperty(IS_ENABLED, DEFAULT_IS_ENABLED); if (this.isEnabled()) { ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); this.destPath = new Path(conf.getProperty(DESTINATION_PATH).get()); } this.isDatePartitioned = conf.getBooleanProperty(IS_DATE_PARTITIONED, DEFAULT_IS_DATE_PARTITIONED); this.writeParallelism = conf.getIntProperty(WRITE_PARALLELISM, DEFAULT_WRITE_PARALLELISM); }
public AwsConfiguration(@NonNull final FileSinkConfiguration conf) { ConfigUtil.checkMandatoryProperties(conf.getConf(), this.getMandatoryProperties()); this.region = conf.getAwsRegion().get(); this.awsAccessKeyId = conf.getAwsAccessKeyId().get(); this.awsSecretAccessKey = conf.getAwsSecretAccesskey().get(); this.bucketName = conf.getBucketName().get(); this.objectKey = conf.getObjectKey().get(); this.sourcePath = conf.getFullPath(); this.fileSystemPrefix = conf.getPathPrefix(); String s3FilePrefix; if (conf.getSourcePartitionPath().isPresent()) { s3FilePrefix = String.format("%s/%s/", this.objectKey, conf.getSourcePartitionPath().get()); } else { s3FilePrefix = String.format("%s/", this.objectKey); } this.pathKey = s3FilePrefix; s3FilePrefix += conf.getFileNamePrefix(); this.s3FilePrefix = s3FilePrefix; }
public HiveConfiguration(@NonNull final Configuration conf) { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); this.dataPath = this.getConf().getProperty(HIVE_DATA_PATH).get(); this.jobName = this.getConf().getProperty(JOB_NAME).get(); this.baseMetadataPath = this.getConf().getProperty(BASE_METADATA_PATH).get(); this.partitionKeyName = this.getConf().getProperty(PARTITION_KEY_NAME); if (this.conf.getProperty(PARTITION_TYPE).isPresent()) { this.partitionType = PartitionType.valueOf(this.getConf().getProperty(PARTITION_TYPE) .get().trim().toUpperCase()); } else { this.partitionType = PartitionType.NONE; } }
public CassandraSinkConfiguration(@NonNull final Configuration conf) { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, this.getMandatoryProperties()); this.partitionKeys = this.splitString(this.conf.getProperty(PARTITION_KEYS).get()); if (this.conf.getProperty(INITIAL_HOSTS).isPresent()) { this.initialHosts = this.splitString(this.conf.getProperty(INITIAL_HOSTS).get()); } else { this.initialHosts = new ArrayList<>(); } // Source fields can be case insensitive so we convert everything to lower case for comparing this.filteredColumns = this.conf.getProperty(COLUMN_LIST).isPresent() ? Optional.of(new HashSet<>(this.splitString(this.conf.getProperty(COLUMN_LIST).get().toLowerCase()))) : Optional.absent(); this.clusteringKeys = this.conf.getProperty(CLUSTERING_KEYS).isPresent() ? initClusterKeys(this.splitString(this.conf.getProperty(CLUSTERING_KEYS).get())) : Collections.EMPTY_LIST; if (this.conf.getProperty(PARTITION_TYPE).isPresent()) { this.partitionType = PartitionType.valueOf(this.conf.getProperty(PARTITION_TYPE) .get().trim().toUpperCase()); } else { this.partitionType = PartitionType.NONE; } this.writeTimestamp = this.conf.getProperty(TIMESTAMP); this.timestampIsLongType = this.conf.getBooleanProperty(TIMESTAMP_IS_LONG_TYPE, false); }
public FileSinkConfiguration(@NonNull final Configuration conf) throws MissingPropertyException, UnsupportedOperationException { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, this.getMandatoryProperties()); this.path = this.conf.getProperty(FS_PATH, DEFAULT_FS_PATH); this.fileType = this.conf.getProperty(FILE_TYPE, DEFAULT_FILE_TYPE);
public LockManagerConfiguration(@NonNull final Configuration conf) { this.conf = conf; this.isEnabled = this.getConf().getBooleanProperty(IS_ENABLED, DEFAULT_IS_ENABLED); if (this.isEnabled()) { ConfigUtil.checkMandatoryProperties(conf, getMandatoryProperties()); this.zkBasePath = cleanZkBasePath(this.getConf().getProperty(ZK_BASE_PATH).get()); } else { this.zkBasePath = null; } this.zkSessionTimeoutMs = this.getConf().getIntProperty(ZK_SESSION_TIMEOUT_MS, DEFAULT_ZK_SESSION_TIMEOUT_MS); this.zkConnectionTimeoutMs = this.getConf().getIntProperty(ZK_CONNECTION_TIMEOUT_MS, DEFAULT_ZK_CONNECTION_TIMEOUT_MS); this.acquireLockTimeMs = this.getConf().getIntProperty(ACQUIRE_LOCK_TIME_MS, DEFAULT_ACQUIRE_LOCK_TIME_MS); }