@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { return errorData.getRawData().toString(); } }
@Override public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
public ConverterResult(@NonNull final ID errorData, @NotEmpty final String errorMessage, @NonNull final OD successData) { this.successData = Optional.of(new ValidData<>(successData)); this.errorData = Optional.of(new ErrorData<>(errorMessage, new RawData<>(errorData))); } }
public ConverterResult(@NonNull final ID errorData, @NotEmpty final String errorMessage) { this.successData = Optional.absent(); this.errorData = Optional.of(new ErrorData<>(errorMessage, new RawData<>(errorData))); }
@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { try { return DatatypeConverter.printHexBinary((byte []) errorData.getRawData().getData()); } catch (Exception e) { log.debug("Not able to retrieve Error source data from ErrorData"); return DEFAULT_ERROR_SOURCE_DATA; } }
@Override public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
public void writeRecordsAndErrors(@NonNull final HoodieWriteResult result, final boolean isErrorTableEnabled) { try { if (result.getException().isPresent()) { throw result.getException().get(); } if (result.getWriteStatuses().isPresent()) { if (isErrorTableEnabled) { // TODO: Can we make this more readable, please? final JavaRDD<Tuple2<HoodieRecord, String>> hoodieRecordAndErrorTupleRDD = result.getWriteStatuses().get() .flatMap(ws -> ws.getFailedRecords().stream().map(fr -> new Tuple2<>(fr, ws.getErrors().get(fr.getKey()).getMessage())).iterator()); final JavaRDD<ErrorData> errorRDD = hoodieRecordAndErrorTupleRDD .map(r -> new ErrorData(r._2, RawDataHelper.getRawData(r._1))); ErrorTableUtil.writeErrorRecordsToErrorTable(this.jsc.sc(), this.hoodieConf.getConf(), Optional.of(this.hoodieConf.getTableName()), new RDDWrapper<>(errorRDD), new HoodieSinkErrorExtractor()); } } } catch (HoodieInsertException | HoodieUpsertException e) { log.error("Error writing to hoodie", e); throw new JobRuntimeException("hoodie write failed :" + (result.getWriteStatuses().isPresent() ? result.getWriteStatuses().get().count() : -1), e); } catch (Exception e) { throw new JobRuntimeException("Error writing to hoodie", e); } }
@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { try { AvroPayload payload = (AvroPayload) errorData.getRawData().getData(); return payload.getData().toString(); } catch (Exception e) { log.debug("Not able to extract Error source data from ErrorData"); return DEFAULT_ERROR_SOURCE_DATA; } }
@Override public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
@Override public String getErrorSourceData(@NonNull final ErrorData errorData) { try { HoodieRecord<HoodieRecordPayload> payload = (HoodieRecord) errorData.getRawData().getData(); String data = String.format("%s. %s", payload.getKey().toString(), String.format("HoodieRecordPayload %s", payload.getData().toString())); return data; } catch (Exception e) { log.debug("Not able to extract Error source data from ErrorData"); return DEFAULT_ERROR_SOURCE_DATA; } }
public String getErrorException(@NonNull final ErrorData errorData) { return errorData.getErrMessage(); } }
private static GenericRecord generateGenericErrorRecord(@NonNull final ErrorExtractor errorExtractor, @NotEmpty final String schema, @NonNull final ErrorData error, @NotEmpty final String applicationId) { Schema errorSchema = new Schema.Parser().parse(schema); GenericRecord newRecord = new GenericData.Record(errorSchema); newRecord.put(HADOOP_ROW_KEY.toLowerCase(), errorExtractor.getRowKey(error.getRawData())); newRecord.put(HADOOP_ERROR_SOURCE_DATA, errorExtractor.getErrorSourceData(error)); newRecord.put(HADOOP_ERROR_EXCEPTION, errorExtractor.getErrorException(error)); newRecord.put(HADOOP_CHANGELOG_COLUMNS.toLowerCase(), errorExtractor.getChangeLogColumns(error.getRawData())); newRecord.put(HADOOP_APPLICATION_ID, applicationId); return newRecord; }
@Test public void convertIncorrectSchema() throws Exception { final String badData = "{\"firstName\": \"Eric\", \"lastName\": \"Sayle\", \"address\": {\"line1\": \"1234 Main St\", \"city\": \"The City\", \"zip\": \"MyZip\"}}"; final Configuration conf = new Configuration(); final Schema schema = new Schema.Parser().parse( getClass().getClassLoader().getResourceAsStream("schemas/schemasource/myTestSchema.1.avsc")); final JSONFileSourceDataConverter converter = new JSONFileSourceDataConverter( conf, new KafkaSourceConverterErrorExtractor(), schema); final List<ConverterResult<String, AvroPayload>> results = converter.convert(badData); Assert.assertEquals(1, results.size()); Assert.assertFalse(results.get(0).getSuccessData().isPresent()); Assert.assertTrue(results.get(0).getErrorData().isPresent()); Assert.assertEquals(badData, results.get(0).getErrorData().get().getRawData().getData()); }