public long getMaxMessagesToRead() { return getConf().getLongProperty(KAFKA_MAX_MESSAGES_TO_READ, DEFAULT_KAFKA_MAX_MESSAGES_TO_READ); } }
public Boolean getUseClientSink() { return this.getConf().getBooleanProperty(USE_CLIENT_SINK, false); }
public Optional<String> getUserName() { return this.getConf().getProperty(USERNAME); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissRegion() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_REGION); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
/** * @return hoodie table name. */ public String getTableName() { return this.getConf().getProperty(getTablePropertyKey(HOODIE_TABLE_NAME, this.tableKey)).get(); }
public static HiveSourceConfiguration initializeConfig(final String jobName, final String dataPath, final String metadataPath) { final Configuration config = new Configuration(); config.setProperty(HiveSourceConfiguration.JOB_NAME, jobName); config.setProperty(HiveSourceConfiguration.HIVE_DATA_PATH, dataPath); config.setProperty(HiveSourceConfiguration.BASE_METADATA_PATH, metadataPath); return new HiveSourceConfiguration(config); } }
@Test(expected = UnsupportedOperationException.class) public void testWithPartitionTypeNoneButPartitionKeyStr() { final Configuration c = initFileNameAndPath("none", false, "version"); c.setProperty(HiveConfiguration.PARTITION_KEY_NAME, "datestr"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
public Builder withInsertParallelism(final int parallelism) { this.conf.setProperty( getTablePropertyKey(HOODIE_INSERT_PARALLELISM, this.tableKey), Integer.toString(parallelism)); return this; }
/** * @return hoodie bulk insert parallelism */ public int getBulkInsertParallelism() { return getProperty(HOODIE_BULKINSERT_PARALLELISM, DEFAULT_HOODIE_PARALLELISM); }
@Test(expected = UnsupportedOperationException.class) public void testConfWithS3DispersalTypeNotSupported() { final Configuration c = initCommon("/newpath/newtest", "csv", 1500, " ", "hdfs", "unknownType"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); } }
@Test(expected = MissingPropertyException.class) public void testWithPartitionTypeButMissSourceSubPath() { final Configuration c = initFileNameAndPath("date", false, "version"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissCredentials() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_ACCESS_KEY_ID); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
/** * @return hoodie metrics prefix. * */ public String getHoodieMetricsPrefix() { return this.getConf().getProperty(getTablePropertyKey(HOODIE_METRICS_PREFIX, this.tableKey)).get(); }
/** * @return hoodie insert parallelism */ public int getInsertParallelism() { return getProperty(HOODIE_INSERT_PARALLELISM, DEFAULT_HOODIE_PARALLELISM); }
public Optional<String> getPassword() { return this.getConf().getProperty(PASSWORD); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissBucketName() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.BUCKET_NAME); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
/** * @return true if {@link com.uber.hoodie.HoodieWriteClient} should rollback inflight commits from previous write * call. */ public boolean shouldRollbackInFlight() { return getProperty(HOODIE_ROLLBACK_INFLIGHT_COMMITS, DEFAULT_HOODIE_ROLLBACK_INFLIGHT_COMMITS); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissObjectKey() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.OBJECT_KEY); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); } }
/** * @return true if insert parallelism needs to be auto tuned. */ public boolean shouldAutoTuneParallelism() { return getProperty(HOODIE_AUTO_TUNE_PARALLELISM, DEFAULT_AUTO_TUNE_PARALLELISM); }
/** * @return hoodie upsert parallelism */ public int getUpsertParallelism() { return getProperty(HOODIE_UPSERT_PARALLELISM, DEFAULT_HOODIE_PARALLELISM); }