private HoodieConfiguration createHoodieConfiguration(@NotEmpty final Configuration conf) { ErrorTableConfiguration errorTableConfiguration = new ErrorTableConfiguration(conf); HoodieConfiguration hoodieConfiguration = errorTableConfiguration.getHoodieConfiguration(conf, testSchema, TARGET_TABLE, ERROR_TABLE, true); return hoodieConfiguration; } }
public static void initErrorTableDataset(@NonNull final Configuration conf, @NotEmpty final String errorTableName) throws IOException { final ErrorTableConfiguration errorTableConf = new ErrorTableConfiguration(conf); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(errorTableName) .withBasePath(errorTableConf.getDestPath().toString()) .withTableName(errorTableName) .enableMetrics(false) .build(); HoodieUtil.initHoodieDataset(FSUtils.getFs(conf), hoodieConf); }
public ErrorTableConfiguration(@NonNull final Configuration conf) { this.conf = conf; this.isEnabled = conf.getBooleanProperty(IS_ENABLED, DEFAULT_IS_ENABLED); if (this.isEnabled()) { ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); this.destPath = new Path(conf.getProperty(DESTINATION_PATH).get()); } this.isDatePartitioned = conf.getBooleanProperty(IS_DATE_PARTITIONED, DEFAULT_IS_DATE_PARTITIONED); this.writeParallelism = conf.getIntProperty(WRITE_PARALLELISM, DEFAULT_WRITE_PARALLELISM); }
final ErrorTableConfiguration errorTableConf = new ErrorTableConfiguration(conf); if (!errorTableConf.isEnabled()) { return; HoodieConfiguration hoodieConf = errorTableConf.getHoodieConfiguration(conf, errorTableSchema, targeTable, errorTable, ERROR_METRICS_IS_ENABLED);
/** * @return hoodie configuration. */ public HoodieConfiguration getHoodieConfiguration(@NonNull final Configuration conf, @NotEmpty final String schema, @NotEmpty final String tableKey, @NotEmpty final String errorTableKey, final boolean errorMetricsEnabled) { final HoodieConfiguration hoodieConf = new HoodieConfiguration(conf, tableKey); final String errorTableName = getErrorTableName(hoodieConf.getTableName()); final HoodieConfiguration.Builder builder = HoodieConfiguration.newBuilder(conf, errorTableKey) .withSchema(schema) .withTableName(errorTableName) .withBasePath(this.getDestPath().toString()) .withBulkInsertParallelism(this.getWriteParallelism()) .enableMetrics(errorMetricsEnabled) .withWriteStatusClass(HoodieWriteStatus.class); // TODO T1793431 fix error metrics and enable metrics if (errorMetricsEnabled) { final String errorMetricsPrefix = getErrorMetricsPrefix(hoodieConf.getHoodieMetricsPrefix()); builder.withMetricsPrefix(errorMetricsPrefix); } return builder.build(); }