public void writeRecordsAndErrors(@NonNull final HoodieWriteResult result, final boolean isErrorTableEnabled) { try { if (result.getException().isPresent()) { throw result.getException().get(); } if (result.getWriteStatuses().isPresent()) { if (isErrorTableEnabled) { // TODO: Can we make this more readable, please? final JavaRDD<Tuple2<HoodieRecord, String>> hoodieRecordAndErrorTupleRDD = result.getWriteStatuses().get() .flatMap(ws -> ws.getFailedRecords().stream().map(fr -> new Tuple2<>(fr, ws.getErrors().get(fr.getKey()).getMessage())).iterator()); final JavaRDD<ErrorData> errorRDD = hoodieRecordAndErrorTupleRDD .map(r -> new ErrorData(r._2, RawDataHelper.getRawData(r._1))); ErrorTableUtil.writeErrorRecordsToErrorTable(this.jsc.sc(), this.hoodieConf.getConf(), Optional.of(this.hoodieConf.getTableName()), new RDDWrapper<>(errorRDD), new HoodieSinkErrorExtractor()); } } } catch (HoodieInsertException | HoodieUpsertException e) { log.error("Error writing to hoodie", e); throw new JobRuntimeException("hoodie write failed :" + (result.getWriteStatuses().isPresent() ? result.getWriteStatuses().get().count() : -1), e); } catch (Exception e) { throw new JobRuntimeException("Error writing to hoodie", e); } }
public final RDDWrapper<OD> map(@NonNull final JavaRDD<ID> data) { final ForkOperator<IData> converter = new ForkOperator<>(data.map(r -> RawDataHelper.getRawData(r)), new DataConversionFunction(), this.conf); converter.execute(); // Write error records. ErrorTableUtil.writeErrorRecordsToErrorTable(data.context(), this.conf, Optional.absent(), new RDDWrapper<>(converter.getRDD(ERROR_RECORD).map(r -> (ErrorData) r), converter.getCount(ERROR_RECORD)), errorExtractor); return new RDDWrapper<>(converter.getRDD(VALID_RECORD).map(r -> ((ValidData<OD>) r).getData()), converter.getCount(VALID_RECORD)); }