private Partitioner getPartitioner(HoodieTable table, boolean isUpsert, WorkloadProfile profile) { if (isUpsert) { return table.getUpsertPartitioner(profile); } else { return table.getInsertPartitioner(profile); } }
public Iterator<List<WriteStatus>> handleUpdate(String commitTime, String fileId, Map<String, HoodieRecord<T>> keyToNewRecords, Optional<HoodieDataFile> dataFileOpt) throws IOException { // these are updates HoodieMergeHandle upsertHandle = getUpdateHandle(commitTime, fileId, keyToNewRecords, dataFileOpt); return handleUpdateInternal(upsertHandle, commitTime, fileId); }
@Override public Iterator<List<WriteStatus>> handleInsertPartition(String commitTime, Integer partition, Iterator recordItr, Partitioner partitioner) { return handleUpsertPartition(commitTime, partition, recordItr, partitioner); }
public static <T extends HoodieRecordPayload> HoodieTable<T> getHoodieTable( HoodieTableMetaClient metaClient, HoodieWriteConfig config, JavaSparkContext jsc) { switch (metaClient.getTableType()) { case COPY_ON_WRITE: return new HoodieCopyOnWriteTable<>(config, jsc); case MERGE_ON_READ: return new HoodieMergeOnReadTable<>(config, jsc); default: throw new HoodieException("Unsupported table type :" + metaClient.getTableType()); } }
public WorkloadProfile(JavaRDD<HoodieRecord<T>> taggedRecords) { this.taggedRecords = taggedRecords; this.partitionPathStatMap = new HashMap<>(); this.globalStat = new WorkloadStat(); buildProfile(); }
@Override public Partitioner getInsertPartitioner(WorkloadProfile profile) { return getUpsertPartitioner(profile); }
/** * Get the view of the file system for this table */ public TableFileSystemView getFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitsTimeline()); }
@Test public void testUpsertPartitioner() throws Exception { final String testPartitionPath = "2016/09/26"; // Inserts + Updates... Check all updates go together & inserts subsplit UpsertPartitioner partitioner = getUpsertPartitioner(0, 200, 100, 1024, testPartitionPath, false); List<HoodieCopyOnWriteTable.InsertBucket> insertBuckets = partitioner.getInsertBuckets(testPartitionPath); assertEquals("Total of 2 insert buckets", 2, insertBuckets.size()); }
@Override public void finalizeWrite(JavaSparkContext jsc, List<HoodieWriteStat> stats) throws HoodieIOException { // delegate to base class for MOR tables super.finalizeWrite(jsc, stats); }
private HoodieWriteConfig.Builder getConfigBuilder(Boolean autoCommit) { return getConfigBuilder(autoCommit, IndexType.BLOOM); }
@Override public Partitioner getUpsertPartitioner(WorkloadProfile profile) { if (profile == null) { throw new HoodieUpsertException("Need workload profile to construct the upsert partitioner."); } mergeOnReadUpsertPartitioner = new MergeOnReadUpsertPartitioner(profile); return mergeOnReadUpsertPartitioner; }
public static <T extends HoodieRecordPayload> HoodieTable<T> getHoodieTable( HoodieTableMetaClient metaClient, HoodieWriteConfig config, JavaSparkContext jsc) { switch (metaClient.getTableType()) { case COPY_ON_WRITE: return new HoodieCopyOnWriteTable<>(config, jsc); case MERGE_ON_READ: return new HoodieMergeOnReadTable<>(config, jsc); default: throw new HoodieException("Unsupported table type :" + metaClient.getTableType()); } }
public Iterator<List<WriteStatus>> handleUpdate(String commitTime, String fileId, Iterator<HoodieRecord<T>> recordItr) throws IOException { // these are updates HoodieMergeHandle upsertHandle = getUpdateHandle(commitTime, fileId, recordItr); return handleUpdateInternal(upsertHandle, commitTime, fileId); }
private Partitioner getPartitioner(HoodieTable table, boolean isUpsert, WorkloadProfile profile) { if (isUpsert) { return table.getUpsertPartitioner(profile); } else { return table.getInsertPartitioner(profile); } }
public WorkloadProfile(JavaRDD<HoodieRecord<T>> taggedRecords) { this.taggedRecords = taggedRecords; this.partitionPathStatMap = new HashMap<>(); this.globalStat = new WorkloadStat(); buildProfile(); }
@Override public Partitioner getInsertPartitioner(WorkloadProfile profile) { return getUpsertPartitioner(profile); }
/** * Get the read optimized view of the file system for this table */ public TableFileSystemView.ReadOptimizedView getROFileSystemView() { return new HoodieTableFileSystemView(metaClient, getCompletedCommitsTimeline()); }
@Override public Iterator<List<WriteStatus>> handleInsertPartition(String commitTime, Integer partition, Iterator recordItr, Partitioner partitioner) { return handleUpsertPartition(commitTime, partition, recordItr, partitioner); }
@Override public void finalizeWrite(JavaSparkContext jsc, List<WriteStatus> writeStatuses) throws HoodieIOException { // delegate to base class for MOR tables super.finalizeWrite(jsc, writeStatuses); }
public Iterator<List<WriteStatus>> handleUpdate(String commitTime, String fileId, Map<String, HoodieRecord<T>> keyToNewRecords, Optional<HoodieDataFile> dataFileOpt) throws IOException { // these are updates HoodieMergeHandle upsertHandle = getUpdateHandle(commitTime, fileId, keyToNewRecords, dataFileOpt); return handleUpdateInternal(upsertHandle, commitTime, fileId); }