/** * @return hoodie configuration. */ public HoodieConfiguration getHoodieConfiguration(@NonNull final Configuration conf, @NotEmpty final String schema, @NotEmpty final String tableKey, @NotEmpty final String errorTableKey, final boolean errorMetricsEnabled) { final HoodieConfiguration hoodieConf = new HoodieConfiguration(conf, tableKey); final String errorTableName = getErrorTableName(hoodieConf.getTableName()); final HoodieConfiguration.Builder builder = HoodieConfiguration.newBuilder(conf, errorTableKey) .withSchema(schema) .withTableName(errorTableName) .withBasePath(this.getDestPath().toString()) .withBulkInsertParallelism(this.getWriteParallelism()) .enableMetrics(errorMetricsEnabled) .withWriteStatusClass(HoodieWriteStatus.class); // TODO T1793431 fix error metrics and enable metrics if (errorMetricsEnabled) { final String errorMetricsPrefix = getErrorMetricsPrefix(hoodieConf.getHoodieMetricsPrefix()); builder.withMetricsPrefix(errorMetricsPrefix); } return builder.build(); }
public static Builder newBuilder(@NonNull final Configuration conf, @NotEmpty final String tableKey) { return new Builder(conf, tableKey); }
new TSBasedHoodieSinkDataConverter(this.conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr) .withCombineBeforeInsert(true).withCombineBeforeUpsert(true).enableMetrics(false).build(); final JavaRDD<AvroPayload> inputRDD = this.jsc.get().parallelize(getRandomData(schemaStr, TS_KEY, RECORD_KEY, 10));
final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(conf, tableName).withTableName(tableName) .withBasePath(basePath.toString()).withSchema(schemaStr).withMetricsPrefix("hoodieMetricsPrefix") .enableMetrics(false).build(); final AtomicBoolean condition = new AtomicBoolean(true); final HoodieBasedMetadataManager mgr =
new TSBasedHoodieSinkDataConverter(this.conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final HoodieBasedMetadataManager hoodieBasedMetadataManager = new HoodieBasedMetadataManager(hoodieConf, new AtomicBoolean(true), this.jsc.get());
new TSBasedHoodieSinkDataConverter(conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final MockHoodieSink hoodieSink = new MockHoodieSink(hoodieConf, hoodieKeyGenerator, jsc.get(), INSERT); final JavaRDD<AvroPayload> inputRDD =
new TSBasedHoodieSinkDataConverter(conf, "firstName", "timestamp", TimeUnit.SECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(conf, ONLY_TABLE) .withSchema(schema.toString()) .build(); final IMetadataManager metadataMgr = new HoodieBasedMetadataManager(hoodieConf, new AtomicBoolean(true), jsc);
new TSBasedHoodieSinkDataConverter(this.conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final HoodieBasedMetadataManager hoodieBasedMetadataManager = new HoodieBasedMetadataManager(hoodieConf, new AtomicBoolean(true), this.jsc.get());
new TSBasedHoodieSinkDataConverter(this.conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final MockHoodieSink hoodieSink = new MockHoodieSink(hoodieConf, hoodieKeyGenerator, jsc.get(), UPSERT); final JavaRDD<AvroPayload> inputRDD =
new TSBasedHoodieSinkDataConverter(this.conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final JavaRDD<AvroPayload> inputRDD = this.jsc.get().parallelize(getRandomData(schemaStr, TS_KEY, RECORD_KEY, 10));
final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(conf, tableName).withTableName(tableName) .withBasePath(basePath.toString()).withSchema(schemaStr).withMetricsPrefix("hoodieMetricsPrefix") .enableMetrics(false).build(); final AtomicBoolean condition = new AtomicBoolean(true); final HoodieBasedMetadataManager mgr = new HoodieBasedMetadataManager(hoodieConf, condition, this.jsc.get());
new TSBasedHoodieSinkDataConverter(conf, RECORD_KEY, TS_KEY, TimeUnit.MILLISECONDS); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final MockHoodieSink hoodieSink = new MockHoodieSink(hoodieConf, hoodieKeyGenerator, jsc.get(), INSERT); final Map<String, String> emptyTags = new HashMap<>();
@Test public void testUpdateBulkInsertParallelism() { final String basePath = "/basePath"; final String tableName = "test-table"; final String schemaStr = getSchema("TS", "RECORD_KEY", 4, 8).toString(); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final HoodieSink mockSink = spy(new HoodieSink(hoodieConf, mock(HoodieSinkDataConverter.class), mock(JavaSparkContext.class), HoodieSink.HoodieSinkOp.NO_OP, new NoOpMetadataManager())); when(mockSink.calculateNewBulkInsertParallelism(anyLong())).thenReturn(18); Assert.assertTrue(mockSink.updateBulkInsertParallelism(1000)); Assert.assertEquals(18, hoodieConf.getBulkInsertParallelism()); Assert.assertEquals(HoodieConfiguration.DEFAULT_HOODIE_PARALLELISM, hoodieConf.getInsertParallelism()); }
@Test public void testUpdateInsertParallelism() { final String basePath = "/basePath"; final String tableName = "test-table"; final String schemaStr = getSchema("TS", "RECORD_KEY", 4, 8).toString(); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(tableName).withTableName(tableName).withMetricsPrefix("test") .withBasePath(basePath).withSchema(schemaStr).enableMetrics(false).build(); final HoodieSink mockSink = spy(new HoodieSink(hoodieConf, mock(HoodieSinkDataConverter.class), mock(JavaSparkContext.class), HoodieSink.HoodieSinkOp.NO_OP, new NoOpMetadataManager())); when(mockSink.calculateNewBulkInsertParallelism(anyLong())).thenReturn(18); Assert.assertTrue(mockSink.updateInsertParallelism(1000)); Assert.assertEquals(18, hoodieConf.getInsertParallelism()); Assert.assertEquals(HoodieConfiguration.DEFAULT_HOODIE_PARALLELISM, hoodieConf.getBulkInsertParallelism()); }
public static void initErrorTableDataset(@NonNull final Configuration conf, @NotEmpty final String errorTableName) throws IOException { final ErrorTableConfiguration errorTableConf = new ErrorTableConfiguration(conf); final HoodieConfiguration hoodieConf = HoodieConfiguration.newBuilder(errorTableName) .withBasePath(errorTableConf.getDestPath().toString()) .withTableName(errorTableName) .enableMetrics(false) .build(); HoodieUtil.initHoodieDataset(FSUtils.getFs(conf), hoodieConf); }
public static HoodieConfiguration createHoodieConfiguration(@NotEmpty final String tableName, @NotEmpty final String basePath) { final String schemaStr = getSchema("ts", "uuid", 4, 8).toString(); return HoodieConfiguration .newBuilder(tableName) .withTableName(tableName) .withBasePath(basePath) .withSchema(schemaStr) .enableMetrics(false) .build(); }