protected HoodieRecordPayload getPayload(@NonNull final AvroPayload payload) { return new HoodieAvroPayload(java.util.Optional.of(payload.getData())); } }
public static List<HoodieRecord<HoodieAvroPayload>> loadFromFile(FileSystem fs, String deltaFilePath, Schema expectedSchema) { List<HoodieRecord<HoodieAvroPayload>> loadedRecords = Lists.newArrayList(); Path path = new Path(deltaFilePath); try { SeekableInput input = new FsInput(path, fs.getConf()); GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(); // Set the expected schema to be the current schema to account for schema evolution reader.setExpected(expectedSchema); FileReader<GenericRecord> fileReader = DataFileReader.openReader(input, reader); for (GenericRecord deltaRecord : fileReader) { String key = deltaRecord.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(); String partitionPath = deltaRecord.get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString(); loadedRecords.add(new HoodieRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Optional.of(deltaRecord)))); } fileReader.close(); // also closes underlying FsInput } catch (IOException e) { throw new HoodieIOException("Could not read avro records from path " + deltaFilePath, e); } return loadedRecords; }
public static List<HoodieRecord> generateHoodieTestRecordsWithoutHoodieMetadata(int from, int limit) throws IOException, URISyntaxException { List<IndexedRecord> iRecords = generateTestRecords(from, limit); return iRecords .stream() .map(r -> new HoodieRecord<>(new HoodieKey(UUID.randomUUID().toString(), "0000/00/00"), new HoodieAvroPayload(Optional.of((GenericRecord) r)))).collect(Collectors.toList()); }
private static HoodieRecord convertToHoodieRecords(IndexedRecord iRecord, String key, String partitionPath) { return new HoodieRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Optional.of((GenericRecord) iRecord))); }
public static List<String> upsertRecords(List<IndexedRecord> iRecords, Map<String, HoodieRecord<? extends HoodieRecordPayload>> records) { List<String> recordKeys = new ArrayList<>(); iRecords .stream() .forEach(r -> { String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(); String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString(); recordKeys.add(key); HoodieRecord record = new HoodieRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Optional.of((GenericRecord) r))); record.setCurrentLocation(new HoodieRecordLocation("DUMMY_COMMIT_TIME", "DUMMY_FILE_ID")); records.put(key, record); }); return recordKeys; } }
public static List<HoodieRecord> updateHoodieTestRecordsWithoutHoodieMetadata(List<HoodieRecord> oldRecords, Schema schema, String fieldNameToUpdate, String newValue) throws IOException, URISyntaxException { return oldRecords .stream() .map(r -> { try { GenericRecord rec = (GenericRecord) r.getData().getInsertValue(schema).get(); rec.put(fieldNameToUpdate, newValue); return new HoodieRecord<>(r.getKey(), new HoodieAvroPayload(Optional.of(rec))); } catch (IOException io) { throw new HoodieIOException("unable to get data from hoodie record", io); } }).collect(Collectors.toList()); }
String iPartitionPath = ((GenericRecord) inMemoryRecord).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString(); HoodieRecord inMemoryHoodieRecord = new HoodieRecord<>(new HoodieKey(ikey, iPartitionPath), new HoodieAvroPayload(Optional.of((GenericRecord) inMemoryRecord))); String dPartitionPath = ((GenericRecord) onDiskRecord).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString(); HoodieRecord onDiskHoodieRecord = new HoodieRecord<>(new HoodieKey(dkey, dPartitionPath), new HoodieAvroPayload(Optional.of((GenericRecord) onDiskRecord)));