/** * @return hoodie base path directory */ public String getBasePath() { // HOODIE_BASE_PATH is a mandatory property. Please check {#getMandatoryProperties()}. return this.getConf().getProperty(getTablePropertyKey(HOODIE_BASE_PATH, this.tableKey)).get(); }
/** * @return hoodie table name. */ public String getTableName() { return this.getConf().getProperty(getTablePropertyKey(HOODIE_TABLE_NAME, this.tableKey)).get(); }
/** * @return hoodie metrics prefix. * */ public String getHoodieMetricsPrefix() { return this.getConf().getProperty(getTablePropertyKey(HOODIE_METRICS_PREFIX, this.tableKey)).get(); }
/** * Ensure that hoodie dataset is present. */ protected void initDataset() { try { HoodieUtil.initHoodieDataset(FSUtils.getFs(this.hoodieConf.getConf()), this.hoodieConf); } catch (IOException e) { log.error("Error initializing hoodie dataset.", e); throw new JobRuntimeException("Could not initialize hoodie dataset", e); } }
/** * It initializes hoodie dataset * @param fs {@link FileSystem} * @param hoodieConf {@link HoodieConfiguration} * @throws IOException */ public static void initHoodieDataset(@NonNull final FileSystem fs, @NonNull final HoodieConfiguration hoodieConf) throws IOException { final Path hoodieMetaFolder = new Path(hoodieConf.getBasePath(), HoodieTableMetaClient.METAFOLDER_NAME); final Path hoodiePropertiesFile = new Path(hoodieMetaFolder.toString(), HoodieTableConfig.HOODIE_PROPERTIES_FILE); if (!fs.exists(hoodiePropertiesFile)) { HoodieTableMetaClient .initializePathAsHoodieDataset(FSUtils.getFs(hoodieConf.getConf()), hoodieConf.getBasePath(), hoodieConf.getHoodieInitProperties()); } }
public void writeRecordsAndErrors(@NonNull final HoodieWriteResult result, final boolean isErrorTableEnabled) { try { if (result.getException().isPresent()) { throw result.getException().get(); } if (result.getWriteStatuses().isPresent()) { if (isErrorTableEnabled) { // TODO: Can we make this more readable, please? final JavaRDD<Tuple2<HoodieRecord, String>> hoodieRecordAndErrorTupleRDD = result.getWriteStatuses().get() .flatMap(ws -> ws.getFailedRecords().stream().map(fr -> new Tuple2<>(fr, ws.getErrors().get(fr.getKey()).getMessage())).iterator()); final JavaRDD<ErrorData> errorRDD = hoodieRecordAndErrorTupleRDD .map(r -> new ErrorData(r._2, RawDataHelper.getRawData(r._1))); ErrorTableUtil.writeErrorRecordsToErrorTable(this.jsc.sc(), this.hoodieConf.getConf(), Optional.of(this.hoodieConf.getTableName()), new RDDWrapper<>(errorRDD), new HoodieSinkErrorExtractor()); } } } catch (HoodieInsertException | HoodieUpsertException e) { log.error("Error writing to hoodie", e); throw new JobRuntimeException("hoodie write failed :" + (result.getWriteStatuses().isPresent() ? result.getWriteStatuses().get().count() : -1), e); } catch (Exception e) { throw new JobRuntimeException("Error writing to hoodie", e); } }
private static Map<String, String> readMetadataInfo( @NonNull final HoodieConfiguration hoodieConf) { try { final FileSystem fs = FSUtils.getFs(hoodieConf.getConf()); HoodieUtil.initHoodieDataset(fs, hoodieConf); final HoodieTableMetaClient hoodieTableMetaClient = new HoodieTableMetaClient(new HadoopConfiguration(hoodieConf.getConf()).getHadoopConf(), hoodieConf.getBasePath(), true); final HoodieActiveTimeline hoodieActiveTimeline = hoodieTableMetaClient.getActiveTimeline(); final java.util.Optional<HoodieInstant> lastInstant = hoodieActiveTimeline.getCommitTimeline() .filterCompletedInstants().lastInstant(); if (lastInstant.isPresent()) { log.info("using hoodie instant for reading checkpoint info :{}", lastInstant.get().getTimestamp()); final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(hoodieActiveTimeline.getInstantDetails(lastInstant.get()).get()); final String serCommitInfo = commitMetadata.getMetadata(HOODIE_METADATA_KEY); if (!Strings.isNullOrEmpty(serCommitInfo)) { return MapUtil.deserializeMap(serCommitInfo); } } return new HashMap<>(); } catch (IOException e) { log.error("failed to read metadata info", e); throw new JobRuntimeException("failed to read metadata information", e); } } }
private static <T> void verifyProperty(@NotEmpty final String tableName, @NonNull final HoodieConfiguration hoodieConf, @NonNull final T defaultValue, @NonNull final T defaultPropertyValue, @NonNull final T tableValue) { Object value = hoodieConf.getProperty(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, defaultValue); Assert.assertTrue(value.equals(defaultValue) && value.getClass() == defaultValue.getClass()); hoodieConf.getConf().setProperty( HoodieConfiguration.getDefaultPropertyKey(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE), defaultPropertyValue.toString()); value = hoodieConf.getProperty(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, defaultValue); Assert.assertTrue( value.equals(defaultPropertyValue) && value.getClass() == defaultPropertyValue.getClass()); hoodieConf.getConf().setProperty( HoodieConfiguration.getTablePropertyKey(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, tableName), tableValue.toString()); value = hoodieConf.getProperty(HoodieConfiguration.HOODIE_INSERT_SPLIT_SIZE, defaultValue); Assert.assertTrue( value.equals(tableValue) && value.getClass() == tableValue.getClass()); } }
Assert.assertTrue( new HoodieTableMetaClient( new HadoopConfiguration(hoodieConf.getConf()).getHadoopConf(), basePath.toString(), true) .getActiveTimeline().getCommitTimeline().filterCompletedInstants().empty()); Assert.assertFalse( new HoodieTableMetaClient( new HadoopConfiguration(hoodieConf.getConf()).getHadoopConf(), basePath.toString(), true) .getActiveTimeline().getCommitTimeline().filterCompletedInstants().empty());
builder.withIndexConfig(new HoodieIndexConfiguration(getConf(), getTableKey()).configureHoodieIndex());