/** * @return List of mandatory properties. */ public List<String> getMandatoryProperties() { return Collections.unmodifiableList(Arrays.asList(getTablePropertyKey(HOODIE_TABLE_NAME, this.tableKey), getTablePropertyKey(HOODIE_BASE_PATH, this.tableKey))); }
public Builder withCombineBeforeUpsert(final boolean combineBeforeUpsert) { this.conf.setProperty(getTablePropertyKey(HOODIE_COMBINE_BEFORE_UPSERT, tableKey), Boolean.toString(combineBeforeUpsert)); return this; }
public Builder enableMetrics(final boolean enableMetrics) { this.conf.setProperty( getTablePropertyKey(HOODIE_ENABLE_METRICS, tableKey), Boolean.toString(enableMetrics)); return this; }
public Builder withBulkInsertParallelism(final int parallelism) { this.conf.setProperty( getTablePropertyKey(HOODIE_BULKINSERT_PARALLELISM, this.tableKey), Integer.toString(parallelism)); return this; }
public Builder withUpsertParallelism(final int parallelism) { this.conf.setProperty( getTablePropertyKey(HOODIE_UPSERT_PARALLELISM, this.tableKey), Integer.toString(parallelism)); return this; }
/** * Used for updating table property */ public void setTableProperty(@NotEmpty final String tablePropertyKey, @NotEmpty final String value) { this.conf.setProperty(getTablePropertyKey(tablePropertyKey, this.tableKey), value); }
public Builder withCombineBeforeInsert(final boolean combineBeforeInsert) { this.conf.setProperty(getTablePropertyKey(HOODIE_COMBINE_BEFORE_INSERT, tableKey), Boolean.toString(combineBeforeInsert)); return this; }
public Builder autoTuneParallelism(final boolean autoTuneParallelism) { this.conf.setProperty( getTablePropertyKey(HOODIE_AUTO_TUNE_PARALLELISM, tableKey), Boolean.toString(autoTuneParallelism)); return this; }
public Builder withWriteStatusClass(@NotEmpty final Class writeStatusClass) { this.conf.setProperty( getTablePropertyKey(HOODIE_WRITE_STATUS_CLASS, tableKey), writeStatusClass.getCanonicalName()); return this; }
public Builder withInsertParallelism(final int parallelism) { this.conf.setProperty( getTablePropertyKey(HOODIE_INSERT_PARALLELISM, this.tableKey), Integer.toString(parallelism)); return this; }
public Builder withMetricsPrefix(@NotEmpty final String metricsPrefix) { this.conf.setProperty(getTablePropertyKey(HOODIE_METRICS_PREFIX, tableKey), metricsPrefix); return this; }
public Builder withBasePath(@NotEmpty final String basePath) { this.conf.setProperty(getTablePropertyKey(HOODIE_BASE_PATH, this.tableKey), basePath); return this; }
public Builder withSchema(@NotEmpty final String schema) { this.conf.setProperty(getTablePropertyKey(HOODIE_AVRO_SCHEMA, this.tableKey), schema); return this; }
public Builder withTableName(@NotEmpty final String tableName) { this.conf.setProperty(getTablePropertyKey(HOODIE_TABLE_NAME, this.tableKey), tableName); return this; }
/** * @return hoodie base path directory */ public String getBasePath() { // HOODIE_BASE_PATH is a mandatory property. Please check {#getMandatoryProperties()}. return this.getConf().getProperty(getTablePropertyKey(HOODIE_BASE_PATH, this.tableKey)).get(); }
/** * @return hoodie table name. */ public String getTableName() { return this.getConf().getProperty(getTablePropertyKey(HOODIE_TABLE_NAME, this.tableKey)).get(); }
/** * @return hoodie metrics prefix. * */ public String getHoodieMetricsPrefix() { return this.getConf().getProperty(getTablePropertyKey(HOODIE_METRICS_PREFIX, this.tableKey)).get(); }
/** * It will read property value from table and default namespace. Value will be returned in following order. * For example for propertyKey ("common.hoodie.%s.insert_split_size") * 1) table specific value ("common.hoodie.tables.table1.insert_split_size" defined in {@link Configuration}) * 2) default hoodie property value ("common.hoodie.default.insert_split_size" defined in {@link Configuration}) * 3) default value specified. (passed in as an argument). * * @param propertyKey hoodie property key * @param defaultValue default value of the property * @param <T> DataType of the property */ public <T> T getProperty(@NotEmpty final String propertyKey, @NonNull final T defaultValue) { final String defaultKey = getDefaultPropertyKey(propertyKey); final String tableKey = getTablePropertyKey(propertyKey, this.tableKey); final T retValue = Configuration.getProperty(this.conf, defaultKey, defaultValue); return Configuration.getProperty(this.conf, tableKey, retValue); }
private static <T> void verifyProperty(@NotEmpty final String tableName, @NonNull final HoodieConfiguration hoodieConf, @NonNull final T defaultValue, @NonNull final T defaultPropertyValue, @NonNull final T tableValue) { Object value = hoodieConf.getProperty(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, defaultValue); Assert.assertTrue(value.equals(defaultValue) && value.getClass() == defaultValue.getClass()); hoodieConf.getConf().setProperty( HoodieConfiguration.getDefaultPropertyKey(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE), defaultPropertyValue.toString()); value = hoodieConf.getProperty(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, defaultValue); Assert.assertTrue( value.equals(defaultPropertyValue) && value.getClass() == defaultPropertyValue.getClass()); hoodieConf.getConf().setProperty( HoodieConfiguration.getTablePropertyKey(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, tableName), tableValue.toString()); value = hoodieConf.getProperty(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, defaultValue); Assert.assertTrue( value.equals(tableValue) && value.getClass() == tableValue.getClass()); } }
@Test public void testErrorTableConfiguration() { final String hoodieTableNameKey = HoodieConfiguration.getTablePropertyKey(HoodieConfiguration.HOODIE_TABLE_NAME, TARGET_TABLE); final String metricsPrefixKey = HoodieConfiguration.getTablePropertyKey(HoodieConfiguration.HOODIE_METRICS_PREFIX, TARGET_TABLE); final String base = HoodieConfiguration.getTablePropertyKey(HoodieConfiguration.HOODIE_BASE_PATH, TARGET_TABLE); Configuration conf = new Configuration(); conf.setProperty(hoodieTableNameKey, tableName); conf.setProperty(metricsPrefixKey, metricsPrefix); conf.setProperty(ErrorTableConfiguration.IS_ENABLED, "true"); conf.setProperty(ErrorTableConfiguration.DESTINATION_PATH, errorBasePath); conf.setProperty(base, basePath); HoodieConfiguration hoodieConfiguration = createHoodieConfiguration(conf); Assert.assertEquals(hoodieConfiguration.getBasePath(), errorBasePath); Assert.assertEquals(hoodieConfiguration.getTableName(), errorTableName); Assert.assertEquals(hoodieConfiguration.getHoodieMetricsPrefix(), errorMetricsPrefix); }