@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { return errorData.getRawData().toString(); } }
public static <T> RawData getRawData(@NonNull final T data) { if (data instanceof byte[]) { return new BinaryRawData((byte[]) data); } return new RawData(data); } }
@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { try { return DatatypeConverter.printHexBinary((byte []) errorData.getRawData().getData()); } catch (Exception e) { log.debug("Not able to retrieve Error source data from ErrorData"); return DEFAULT_ERROR_SOURCE_DATA; } }
public final RDDWrapper<OD> map(@NonNull final JavaRDD<ID> data) { final ForkOperator<IData> converter = new ForkOperator<>(data.map(r -> RawDataHelper.getRawData(r)), new DataConversionFunction(), this.conf); converter.execute(); // Write error records. ErrorTableUtil.writeErrorRecordsToErrorTable(data.context(), this.conf, Optional.absent(), new RDDWrapper<>(converter.getRDD(ERROR_RECORD).map(r -> (ErrorData) r), converter.getCount(ERROR_RECORD)), errorExtractor); return new RDDWrapper<>(converter.getRDD(VALID_RECORD).map(r -> ((ValidData<OD>) r).getData()), converter.getCount(VALID_RECORD)); }
/** * It is used for filtering out tupleEntries. If it returns true then tupleEntry will be * filtered out. It will have same set of keys as defined by corresponding ForkFunction. * * @param forkData : forkData to be filtered out or retained. */ protected Boolean execute(final ForkData<DI> forkData) { return forkData.getKeys().contains(this.filterKey); } }
@Override public String getRowKey(@NonNull final RawData rawdata) { try { return (String) ((AvroPayload) rawdata.getData()).getField(ErrorTableUtil.HADOOP_ROW_KEY); } catch (Exception e) { log.debug("Not able to extract Hadoop_row_key from RawData"); return DEFAULT_ROW_KEY; } }
@Override public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
@Override public String toString() { return DatatypeConverter.printHexBinary(getData()); } }
/** * Constructor for case that OD is schema conforming * @param successData */ public ConverterResult(@NonNull final OD successData) { this.successData = Optional.of(new ValidData<>(successData)); this.errorData = Optional.absent(); }
public JavaRDD<DI> getRDD(final FilterFunction<DI> filterFunction) { if (!this.groupRDD.isPresent()) { throw new ForkOperationException("No RDD is found"); } return this.groupRDD.get().filter(filterFunction).map(record -> record.getRecord()); }
@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { try { AvroPayload payload = (AvroPayload) errorData.getRawData().getData(); return payload.getData().toString(); } catch (Exception e) { log.debug("Not able to extract Error source data from ErrorData"); return DEFAULT_ERROR_SOURCE_DATA; } }
@Override public final Iterator<ForkData<DI>> call(final DI di) { final List<ForkData<DI>> forkData = process(di); forkData.stream().forEach(fd -> verifyKeys(fd.getKeys(), di)); return forkData.iterator(); }
@Override public String getRowKey(@NonNull final RawData rawdata) { try { return ((HoodieRecord) rawdata.getData()).getKey().getRecordKey(); } catch (Exception e) { log.debug("Not able to extract Hadoop_row_key from RawData"); return DEFAULT_ROW_KEY; } }
@Override public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { try { HoodieRecord<HoodieRecordPayload> payload = (HoodieRecord) errorData.getRawData().getData(); String data = String.format("%s. %s", payload.getKey().toString(), String.format("HoodieRecordPayload %s", payload.getData().toString())); return data; } catch (Exception e) { log.debug("Not able to extract Error source data from ErrorData"); return DEFAULT_ERROR_SOURCE_DATA; } }
@Override protected List<ForkData<String>> process(final String record) { final List<ForkData<String>> forkData = super.process(record); // Add invalid keys. forkData.stream().forEach(fd -> fd.getKeys().add(UNREGISTERED_KEY)); return forkData; } }
@Override public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }