/** * Get only the completed (no-inflights) commit + deltacommit timeline */ public HoodieTimeline getCompletedCommitsTimeline() { return metaClient.getCommitsTimeline().filterCompletedInstants(); }
/** * Get the completed (commit + compaction) view of the file system for this table */ public TableFileSystemView getCompletedFileSystemView() { return new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline()); }
/** * Get only the inflights (no-completed) commit timeline */ public HoodieTimeline getInflightCommitTimeline() { return metaClient.getCommitsTimeline().filterInflightsExcludingCompaction(); }
/** * Get only the completed (no-inflights) commit timeline */ public HoodieTimeline getCompletedCommitTimeline() { return metaClient.getCommitsTimeline().filterCompletedInstants(); }
/** * Get only the inflights (no-completed) commit timeline */ public HoodieTimeline getInflightCommitTimeline() { return metaClient.getCommitsTimeline().filterInflightsExcludingCompaction(); }
/** * Get the completed (commit + compaction) view of the file system for this table */ public TableFileSystemView getCompletedFileSystemView() { return new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline()); }
/** * Cleanup all inflight commits */ private void rollbackInflightCommits() { HoodieTable<T> table = HoodieTable.getHoodieTable( new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); HoodieTimeline inflightTimeline = table.getMetaClient().getCommitsTimeline().filterInflightsExcludingCompaction(); List<String> commits = inflightTimeline.getInstants().map(HoodieInstant::getTimestamp) .collect(Collectors.toList()); Collections.reverse(commits); for (String commit : commits) { rollback(commit); } }
/** * Cleanup all inflight commits */ private void rollbackInflightCommits() { HoodieTable<T> table = HoodieTable.getHoodieTable( new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); HoodieTimeline inflightTimeline = table.getMetaClient().getCommitsTimeline().filterInflightsExcludingCompaction(); List<String> commits = inflightTimeline.getInstants().map(HoodieInstant::getTimestamp) .collect(Collectors.toList()); Collections.reverse(commits); for (String commit : commits) { rollback(commit); } }
java.util.Optional<HoodieInstant> latestCommitTime = hoodieTable.getMetaClient().getCommitsTimeline() .filterCompletedInstants().lastInstant(); List<Tuple2<String, HoodieDataFile>> filteredFiles = new ArrayList<>();
java.util.Optional<HoodieInstant> latestCommitTime = hoodieTable.getMetaClient().getCommitsTimeline() .filterCompletedInstants().lastInstant(); List<Tuple2<String, HoodieDataFile>> filteredFiles = new ArrayList<>();
/** * Reads the paths under the a hoodie dataset out as a DataFrame */ public static Dataset<Row> read(JavaSparkContext jsc, String basePath, SQLContext sqlContext, FileSystem fs, String... paths) { List<String> filteredPaths = new ArrayList<>(); try { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs.getConf(), basePath, true); for (String path : paths) { TableFileSystemView.ReadOptimizedView fileSystemView = new HoodieTableFileSystemView( metaClient, metaClient.getCommitsTimeline().filterCompletedInstants(), fs.globStatus(new Path(path))); List<HoodieDataFile> latestFiles = fileSystemView.getLatestDataFiles().collect(Collectors.toList()); for (HoodieDataFile file : latestFiles) { filteredPaths.add(file.getPath()); } } return sqlContext.read().parquet(filteredPaths.toArray(new String[filteredPaths.size()])); } catch (Exception e) { throw new HoodieException("Error reading hoodie dataset as a dataframe", e); } }
HoodieTimeline timeline = table.getMetaClient().getCommitsTimeline();
metaClient.getCommitsTimeline().filterCompletedInstants(), allFiles); Stream<HoodieDataFile> dataFilesToRead = roView.getLatestDataFiles(); Map<String, Long> parquetFileIdToSize = dataFilesToRead.collect(
metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); hoodieTable = HoodieTable.getHoodieTable(metaClient, cfg, jsc); roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline(), allFiles); List<HoodieDataFile> dataFiles2 = roView.getLatestDataFiles().collect(Collectors.toList()); metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); hoodieTable = HoodieTable.getHoodieTable(metaClient, cfg, jsc); roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline(), allFiles);