private static Configuration getValidHiveSourceConfiguration() { final Configuration config = new Configuration(); config.setProperty(HiveSourceConfiguration.JOB_NAME, JOB_NAME); config.setProperty(HiveSourceConfiguration.HIVE_DATA_PATH, DEFAULT_DATA_PATH); config.setProperty(HiveSourceConfiguration.BASE_METADATA_PATH, DEFAULT_METADATA_PATH); return config; } }
public static <T> T getProperty(@NonNull final Configuration conf, @NotEmpty final String key, @NonNull final T defaultValue) { if (defaultValue instanceof Integer) { return (T) new Integer(conf.getIntProperty(key, ((Integer) defaultValue).intValue())); } else if (defaultValue instanceof Long) { return (T) new Long(conf.getLongProperty(key, ((Long) defaultValue).longValue())); } else if (defaultValue instanceof String) { return (T) conf.getProperty(key, (String) defaultValue); } else if (defaultValue instanceof Double) { return (T) new Double(conf.getDoubleProperty(key, ((Double) defaultValue).doubleValue())); } else if (defaultValue instanceof Boolean) { return (T) new Boolean(conf.getBooleanProperty(key, ((Boolean) defaultValue).booleanValue())); } else { throw new IllegalArgumentException("Not supported :" + defaultValue.getClass()); } }
/** * Returns hadoop configuration. */ public org.apache.hadoop.conf.Configuration getHadoopConf() { final org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); this.conf.getPropertiesWithPrefix(HADOOP_COMMON_PROPERTY_PREFIX, true).forEach( (key, value) -> { log.info("hadoop-conf-update:key:[{}]:value:[{}]", key, value); hadoopConf.set(key, value); } ); return hadoopConf; } }
public Optional<Long> getTimeToLive() { return this.conf.getProperty(TIME_TO_LIVE).isPresent() ? Optional.of(this.conf.getLongProperty(TIME_TO_LIVE, DEFAULT_TIME_TO_LIVE)) : Optional.absent(); }
public ErrorTableConfiguration(@NonNull final Configuration conf) { this.conf = conf; this.isEnabled = conf.getBooleanProperty(IS_ENABLED, DEFAULT_IS_ENABLED); if (this.isEnabled()) { ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); this.destPath = new Path(conf.getProperty(DESTINATION_PATH).get()); } this.isDatePartitioned = conf.getBooleanProperty(IS_DATE_PARTITIONED, DEFAULT_IS_DATE_PARTITIONED); this.writeParallelism = conf.getIntProperty(WRITE_PARALLELISM, DEFAULT_WRITE_PARALLELISM); }
@Test public void loadConfigurationYaml() { final Configuration conf = new Configuration(new File(CONFIG_YAML), Optional.absent()); conf.setProperty("marmaray.string_configs.string3", "string3"); Assert.assertEquals(1, conf.getIntProperty("marmaray.scalar_configs.integer", -1)); Assert.assertEquals(1234567890123L, conf.getLongProperty("marmaray.scalar_configs.long", -1)); Assert.assertEquals(1.23, conf.getDoubleProperty("marmaray.scalar_configs.double", -1.0), DELTA); Assert.assertEquals(true, conf.getBooleanProperty("marmaray.scalar_configs.boolean", false)); Assert.assertEquals("string1", conf.getProperty("marmaray.string_configs.string1", "not string1")); Assert.assertEquals("string2", conf.getProperty("marmaray.string_configs.stringlist.string2").get()); final Map<String, String> configMap = conf.getPropertiesWithPrefix("marmaray.string_configs.", true); Assert.assertEquals("string2", configMap.get("stringlist.string2")); Assert.assertEquals("string3", configMap.get("string3")); Assert.assertEquals(-1, conf.getIntProperty("marmaray.scalar_configs.notinteger", -1)); Assert.assertEquals(-1, conf.getLongProperty("marmaray.scalar_configs.notlong", -1)); Assert.assertEquals(-1.0, conf.getDoubleProperty("marmaray.scalar_configs.notdouble", -1.0), DELTA); Assert.assertEquals(false, conf.getBooleanProperty("marmaray.scalar_configs.notboolean", false)); Assert.assertEquals("not string1", conf.getProperty("marmaray.string_configs.notstring1", "not string1")); }
public static Builder newBuilder(@NotEmpty final String tableKey) { return newBuilder(new Configuration(), tableKey); }
public Optional<String> getNativePort() { return this.conf.getProperty(NATIVE_TRANSPORT_PORT); }
public Builder withInsertParallelism(final int parallelism) { this.conf.setProperty( getTablePropertyKey(HOODIE_INSERT_PARALLELISM, this.tableKey), Integer.toString(parallelism)); return this; }
this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, this.getMandatoryProperties()); this.path = this.conf.getProperty(FS_PATH, DEFAULT_FS_PATH); this.fileType = this.conf.getProperty(FILE_TYPE, DEFAULT_FILE_TYPE); this.fileSizeMegaBytes = this.conf.getDoubleProperty(FILE_SIZE_MEGABYTE, DEFAULT_FILE_SIZE); this.columnHeader = this.conf.getBooleanProperty(CSV_COLUMN_HEADER, DEFAULT_CSV_COLUMN_HEADER); this.pathPrefix = this.conf.getProperty(PATH_PREFIX).get(); if (this.conf.getProperty(SEPARATOR).isPresent()) { if (this.conf.getProperty(SEPARATOR).get().length() != 1) { throw new UnsupportedOperationException("The separator should only contain one single character."); } else { this.separator = this.conf.getProperty(SEPARATOR).get().charAt(0); if (this.conf.getProperty(PARTITION_TYPE).isPresent()) { this.partitionType = PartitionType.valueOf(this.conf.getProperty(PARTITION_TYPE) .get().trim().toUpperCase()); } else { if (this.conf.getProperty(DISPERSAL_TYPE).isPresent()) { final String dispersalType = this.conf.getProperty(DISPERSAL_TYPE).get().trim().toUpperCase(); final Boolean isValid = EnumUtils.isValidEnum(DispersalType.class, dispersalType); if (isValid) { if (this.conf.getProperty(FILE_SINK_TYPE).isPresent()) { final String sinkName = this.conf.getProperty(FILE_SINK_TYPE).get().trim().toUpperCase(); final Boolean isValid = EnumUtils.isValidEnum(FileSinkType.class, sinkName); if (isValid) {
public HiveSourceConfiguration(@NonNull final Configuration conf) { super(conf); this.saveCheckpoint = this.getConf().getBooleanProperty(SAVE_CHECKPOINT, true); this.startDate = getConf().getProperty(HIVE_START_DATE).isPresent() ? Optional.of(DateTime.parse(getConf().getProperty(HIVE_START_DATE).get(), DateTimeFormat.forPattern(HIVE_START_DATE_FORMAT).withZoneUTC()).toDate()) : Optional.absent(); }
public void loadYamlFile(@NonNull final File yamlFile, final Optional<String> scope) { try { final FileSystem localFs = FileSystem.getLocal( new HadoopConfiguration(new Configuration()).getHadoopConf()); final InputStream yamlInputStream = localFs.open(new Path(yamlFile.getPath())); loadYamlStream(yamlInputStream, scope); } catch (IOException e) { final String errorMsg = String .format("Error loading yaml config file %s", yamlFile.getAbsolutePath()); log.error(errorMsg, e); throw new JobRuntimeException(errorMsg, e); } }
@Test public void testConfigurationParseWithScope() { final Configuration conf = new Configuration(scopeAwareConfigInputStream, Optional.of("bootstrap")); Assert.assertEquals("jdbc", conf.getProperty("database.connection.type").get()); Assert.assertEquals("5000", conf.getProperty("database.connection.port").get()); Assert.assertEquals("4000", conf.getProperty("hbase.connection.port").get()); Assert.assertEquals("1", conf.getProperty("marmaray.hoodie.tables.non_primitive_field_from_default.hello") .get()); Assert.assertEquals("2", conf.getProperty("marmaray.hoodie.tables.non_primitive_field_from_default.hi").get()); Assert.assertEquals("3", conf.getProperty("marmaray.hoodie.tables.non_primitive_field_from_bootstrap.heya") .get()); Assert.assertEquals("1000", conf.getProperty("marmaray.hoodie.tables.target_table.parallelism").get()); Assert.assertEquals("false", conf.getProperty("marmaray.hoodie.tables.target_table.combine_before_insert").get()); Assert.assertEquals("2147483647", conf.getProperty("marmaray.hoodie.tables.target_table.parquet_max_file_size").get()); }
/** * Returns hadoop configuration. * Some example of things that could be set are the output rpc port (cassandra.output.thrift.port) * and other cassandra specific hadoop configuration settings. */ public org.apache.hadoop.conf.Configuration getHadoopConf() { final org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); if (this.getUserName().isPresent() && this.getPassword().isPresent()) { ConfigHelper.setOutputKeyspaceUserNameAndPassword( hadoopConf, this.getUserName().get(), this.getPassword().get() ); } if (!this.conf.getProperty(DATACENTER).isPresent()) { // should never happen, but a sanity check throw new MissingPropertyException("The datacenter information is missing"); } // By default use Murmur3 Partitioner for now, we may want this to be changed in future ConfigHelper.setOutputPartitioner(hadoopConf, Murmur3Partitioner.class.getName()); log.info("Setting output local DC only to true"); ConfigHelper.setOutputLocalDCOnly(hadoopConf, true); this.conf.getPropertiesWithPrefix(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX, true).forEach( (key, value) -> { log.info("hadoop-conf-update:key:[{}]:value:[{}]", key, value); hadoopConf.set(key, value); } ); return hadoopConf; }
public int getNumRetries() { return conf.getIntProperty(NUM_OF_RETRIES, SimpleRetryStrategy.DEFAULT_NUMBER_OF_RETRIES); }
public long getWaitTimeInMs() { return conf.getLongProperty(WAIT_TIME_IN_MS, SimpleRetryStrategy.DEFAULT_WAIT_TIME_IN_MS); } }
public void loadYamlStream(@NonNull final InputStream yamlStream, @NonNull final Optional<String> scope) { try { final ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory()); final JsonNode jsonNode = yamlReader.readTree(yamlStream); final JsonNode scopeOverriddenJsonNode = handleScopeOverriding(scope, jsonNode); parseConfigJson(scopeOverriddenJsonNode, ""); } catch (IOException e) { final String errorMsg = "Error loading yaml file "; log.error(errorMsg, e); throw new JobRuntimeException(errorMsg, e); } }
public Boolean getUseClientSink() { return this.getConf().getBooleanProperty(USE_CLIENT_SINK, false); }
public DummyHoodieSinkDataConverter() { super(new Configuration(), new ErrorExtractor()); }
public String getSchema() { return this.conf.getProperty(SCHEMA).get(); }